The goal of this notebook is to test the endpoints exposed by CyNDEx2 by importing, exporting, modifying, and updating singletons and collections via CyREST. The roundtrip (of collections AND singletons with modifications on either side) must preserve network and collection level attributes (opaque or not).
In [1]:
# import modules for network creation
from py2cytoscape.data.cyrest_client import CyRestClient
import numpy as np
import requests, json, time
# Test accounts to use
ndex_accounts = {
'public':
{'serverUrl': 'http://ndexbio.org/v2', 'username':'bsettle', 'password':'ndexTest'},
'test':
{'serverUrl': 'http://dev.ndexbio.org/v2', 'username':'bsettle', 'password':'ndexTest'}
}
REST_ENDPOINT = 'http://localhost:1234'
In [2]:
'''def getUserId(acc):
url = ndex_accounts[acc]['serverUrl'] + '/user?username=' + ndex_accounts[acc]['username']
resp = requests.get(url)
if 'externalId' in resp.json():
return resp.json()['externalId']
'''
def assertState(attributes, state):
for k in attributes:
if k not in state or state[k] != attributes[k]:
print("{} not in {}".format(k, state))
return False
return True
def addNetworkAttributes(SUID, data):
data = data.copy()
resp = requests.get(REST_ENDPOINT + "/v1/networks/{}/tables/defaultnetwork".format(SUID))
net_table = resp.json()
data.update({"SUID": SUID})
new_data = {
"key": "SUID",
"dataKey": "SUID",
"data": [
data
]
}
resp = requests.put(REST_ENDPOINT + '/v1/networks/{}/tables/defaultnetwork'.format(SUID), data=json.dumps(new_data))
resp = requests.get(REST_ENDPOINT + "/v1/networks/{}/tables/defaultnetwork".format(SUID))
return resp.json()
def addCollectionAttributes(SUID, data):
data = data.copy()
resp = requests.get(REST_ENDPOINT + "/v1/collections/{}/tables/default".format(SUID))
net_table = resp.json()
data.update({"SUID": SUID})
new_data = {
"key": "SUID",
"dataKey": "SUID",
"data": [
data
]
}
resp = requests.put(REST_ENDPOINT + '/v1/collections/{}/tables/default'.format(SUID), data=json.dumps(new_data))
resp = requests.get(REST_ENDPOINT + "/v1/collections/{}/tables/default".format(SUID))
return resp.json()
def getCurrentNetworks():
data = requests.get(REST_ENDPOINT + "/cyndex2/v1/networks/current").json()
return data['data']
def uploadNetwork(suid, data, method='post'):
if method == 'post':
resp = requests.post(REST_ENDPOINT + '/cyndex2/v1/networks/%s' % suid,
data=json.dumps(data), headers={'Content-Type': 'application/json'})
else:
resp = requests.put(REST_ENDPOINT + '/cyndex2/v1/networks/%s' % suid,
data=json.dumps(data), headers={'Content-Type': 'application/json'})
try:
uuid = resp.json()['data']['uuid']
except Exception as e:
raise Exception(resp.content)
return uuid
def importNetwork(uuid, account={'serverUrl': 'http://ndexbio.org/v2'}):
data = {'uuid': uuid}
data.update(account)
resp = requests.post(REST_ENDPOINT + '/cyndex2/v1/networks',
data=json.dumps(data), headers={'Content-Type': 'application/json'})
data=resp.json()['data']
return data['suid'], data['uuid']
In [3]:
def compareAttrs(cyData, ndexData, attrs='all'):
'''Compare dicts for overlapping pairs. Prints attributes to be compared, and to be excluded
Throws an exception if any attributes to be compared are different'''
if attrs == 'all':
attrs = [k for k in cyData if k in ndexData]
print("Attributes being compared: {}".format(attrs))
print("Excluded cyData: {}".format([k for k in cyData if k not in ndexData]))
print("Excluded ndexData: {}".format([k for k in ndexData if k not in cyData]))
for attr in attrs:
assert attr in cyData, 'Attribute {} not in cytoscape data'.format(attr)
assert attr in ndexData, 'Attribute {} not in NDEx data'.format(attr)
assert cyData[attr] == ndexData[attr], "Attribute {} invalid. Cy:{} != NDEx:{}".format(attr, cyData[attr], ndexData[attr])
In [4]:
def compareCollectionAttributes(suid, uuid, attrs='all', subAttrs='all', ndexUrl="http://ndexbio.org/v2"):
'''Compare the collection and its subnetworks in Cytoscape with its NDEx sibling by reaching CyREST
and NDEx endpoints. Subnetworks are compared with the compareNetworkAttributes function
Prints attributes to be compared, and to be excluded
Throws an exception if any attributes to be compared are different
'''
resp = requests.get(REST_ENDPOINT + "/v1/collections/{}/tables/default".format(suid))
cyData = resp.json()['rows'][0]
cySubData = {}
subs = requests.get(REST_ENDPOINT + "/v1/collections/{}/subnetworks".format(suid)).json()
for sub in subs:
resp = requests.get(REST_ENDPOINT + "/v1/networks/{}/tables/defaultnetwork".format(sub))
cySubData[sub] = resp.json()['rows'][0]
ndexSubData = {}
ndexData = {}
resp = requests.get(ndexUrl + '/network/{}/aspect/networkAttributes'.format(uuid))
ndexCollData = resp.json()
for data in ndexCollData:
val = float(data['v']) if 'd' in data and data['d'] == 'double' else data['v']
if 's' in data:
if data['s'] not in ndexSubData:
ndexSubData[data['s']] = {}
ndexSubData[data['s']][data['n']] = val
else:
ndexData[data['n']] = val
print("Comparing collection level")
compareAttrs(cyData, ndexData, attrs=attrs)
print("Comparing subnetworks")
assert set(cySubData.keys()) == set(ndexSubData.keys()), "Subnetworks are different, Cy:{} != NDEx:{}".format(cySubData.keys(), ndexSubData.keys())
for n in cySubData:
attrs = subAttrs[n] if (type(subAttrs) == dict and n in subAttrs) else 'all'
compareAttrs(cySubData[n], ndexSubData[n], attrs=attrs)
In [5]:
def compareNetworkAttributes(suid, uuid, attrs='all', ndexUrl="http://ndexbio.org/v2"):
'''Compare the subnetwork in Cytoscape with its NDEx sibling by reaching CyREST and NDEx endpoints
Prints attributes to be compared, and to be excluded
Throws an exception if any attributes to be compared are different
'''
resp = requests.get(REST_ENDPOINT + "/v1/networks/{}/tables/defaultnetwork".format(suid))
cyData = resp.json()['rows'][0]
resp = requests.get(ndexUrl + '/network/{}/aspect/networkAttributes'.format(uuid))
ndexData = resp.json()
ndexData = {d['n']: float(d['v']) if 'd' in d and d['d'] == 'double' else d['v'] for d in ndexData}
compareAttrs(cyData, ndexData, attrs=attrs)
In [6]:
def checkNetworkInNDEx(suid, **args):
''' Check the specified network/collection in Cytoscape for a UUID, and compare the attributes with its sibline
on NDEx.
Returns true if the shared attributes match, false otherwise
Arguments:
suid: SUID of network or collection
ndexUrl: server address if it is not http://ndexbio.org/v2
attrs: Attributes to compare, defaults to only shared attrs
subAttrs: Attributes to compare in subnetworks, defaults to shared
'''
info = requests.get(REST_ENDPOINT + '/cyndex2/v1/networks/{}'.format(suid)).json()['data']
if 'currentNetworkSuid' in info:
for net in info['members']:
if net['suid'] == suid and 'uuid' in net:
compareNetworkAttributes(net['suid'], net['uuid'], **args)
return True
else:
if 'uuid' in info['currentRootNetwork']:
compareCollectionAttributes(info['currentRootNetwork']['suid'], info['currentRootNetwork']['uuid'], **args)
return True
return False
In [7]:
print("Importing ACSN: Protein-Protein Interactions from NDEx")
collectionSUID, uuid = importNetwork("34f29fd1-884b-11e7-a10d-0ac135e8bacf")
print("Asserting network not imported as collection")
assert not checkNetworkInNDEx(collectionSUID), "Collection has an SUID {}, but subnetwork should have it".format(suid)
print("Checking that the network was imported as singleton with all attributes")
subs = requests.get(REST_ENDPOINT + "/v1/collections/{}/subnetworks".format(collectionSUID)).json()
assert checkNetworkInNDEx(subs[0]), "CyNetwork with SUID {} not equal to NDEx net with UUID {}".format(subs[0], uuid)
print("Saving as singleton")
data = {'isPublic': True, 'metadata': {}}
data.update(ndex_accounts['test'])
newUUID = uploadNetwork(subs[0], data)
print("Saved network with UUID {}\n".format(newUUID))
print("Checking new network on NDEx")
assert checkNetworkInNDEx(subs[0], ndexUrl="http://dev.ndexbio.org/v2"), "CyNetwork with SUID {} not equal to NDEx net with UUID {}".format(subs[0], uuid)
print("Adding attributes to subnetwork")
addNetworkAttributes(subs[0], {'extraVal': 'newValue2'})
newUUID = uploadNetwork(subs[0], data, method='put')
print("Updated network with UUID {}\n".format(newUUID))
print("Checking updated network on NDEx")
assert checkNetworkInNDEx(subs[0], attrs=['extraVal'], ndexUrl="http://dev.ndexbio.org/v2"), "CyNetwork with SUID {} not equal to NDEx net with UUID {} after update".format(subs[0], uuid)
print("Cleaning up")
resp = requests.delete(REST_ENDPOINT + '/v1/networks/{}'.format(subs[0]))
In [8]:
'''
Create collection "Collection" in cytoscape with 2 subnetworks "SubA" and "SubB"
Populate "description" column in each with "collection desc", "subA desc", and "subB desc", respectively
Save node and edge data for comparison later
'''
print("Initializing client...")
# Create REST client for Cytoscape
cy = CyRestClient()
# Reset current session for fresh start
cy.session.delete()
print("Creating Collection")
# Prepare ndarray data
matrixA = np.array([
[0, 1, 1, 0],
[0, 0, 1, 0],
[0, 0, 0, 1],
[0, 0, 0, 0]
])
matrixB = np.array([
[0, 1, 1, 1, 0],
[0, 0, 0, 0, 1],
[0, 0, 0, 1, 0],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 0],
])
# Generate cytoscape network obejct from ndarray
subA = cy.network.create_from_ndarray(matrixA, name='SubA', collection='Collection')
subB = cy.network.create_from_ndarray(matrixB, name='SubB', collection='Collection')
collectionSUID = requests.get(REST_ENDPOINT + "/v1/collections").json()[0]
subASUID = subA._CyNetwork__id
subBSUID = subB._CyNetwork__id
print("Collection with SUID %s" % collectionSUID)
print("SubA with SUID %s" % subASUID)
print("SubB with SUID %s" % subBSUID)
print("Applying layout...")
cy.layout.apply(network=subA)
cy.layout.apply(network=subB)
print("Networks created")
network_states = {
collectionSUID: {},
subASUID: {},
subBSUID: {},
}
node_data = {
subASUID: {},
subBSUID: {}
}
print("Adding to SubA {author: BrettA, description: descA}")
subAData = {"author": "BrettA", "description": "descA"}
respA = addNetworkAttributes(subASUID, subAData)
network_states[subASUID].update(subAData)
print("Adding to SubB {author: BrettB, description: descB}")
subBData = {"author": "BrettB", "description": "descB"}
respB = addNetworkAttributes(subBSUID, subBData)
network_states[subBSUID].update(subBData)
print("Adding extra column to subA nodes")
tableA = subA.get_node_table()
extra = ['a', 'b', 'c', 'd']
tableA['extra'] = extra
node_data[subASUID] = {'extra': extra}
subA.update_node_table(tableA, network_key_col='name', data_key_col='name')
print("Adding to Collection {author: Brett Coll, disease: example}")
collData = {'author': 'Brett Coll', 'disease': 'example'}
resp = addCollectionAttributes(collectionSUID, collData)
network_states[collectionSUID].update(collData)
In [9]:
print("Saving as collection")
data = {'isPublic': True, 'metadata': {}}
data.update(ndex_accounts['test'])
newUUID = uploadNetwork(collectionSUID, data)
print("Saved collection with UUID {}\n".format(newUUID))
print("Checking new collection on NDEx")
assert checkNetworkInNDEx(collectionSUID, ndexUrl="http://dev.ndexbio.org/v2"), "CyNetwork with SUID {} not equal to NDEx net with UUID {}".format(collectionSUID, uuid)
print("Adding attributes to subnetwork")
subs = requests.get(REST_ENDPOINT + "/v1/collections/{}/subnetworks".format(collectionSUID)).json()
addNetworkAttributes(subs[0], {'extraSubVal': 'newValue2'})
print("Adding attributes to collection")
addCollectionAttributes(collectionSUID, {'extraColVal': 'newValue2'})
newUUID = uploadNetwork(collectionSUID, data, method='put')
print("Updated collection with UUID {}\n".format(newUUID))
print("Checking updated collection on NDEx")
subAttrs = {subs[0]: ['extraSubVal']}
assert checkNetworkInNDEx(collectionSUID, attrs=['extraColVal'], subAttrs=subAttrs, ndexUrl="http://dev.ndexbio.org/v2"), "CyNetwork with SUID {} not equal to NDEx net with UUID {} after update".format(collectionSUID, uuid)
print("Cleaning up")
resp = requests.delete(REST_ENDPOINT + '/v1/networks/{}'.format(subs[0]))
resp = requests.delete(REST_ENDPOINT + '/v1/networks/{}'.format(subs[1]))
In [11]:
'''
Create collection "Collection" in cytoscape with 2 subnetworks "SubA" and "SubB"
Populate "description" column in each with "collection desc", "subA desc", and "subB desc", respectively
Save node and edge data for comparison later
'''
print("Initializing client...")
# Create REST client for Cytoscape
cy = CyRestClient()
# Reset current session for fresh start
cy.session.delete()
print("Creating Collection")
# Prepare ndarray data
matrixA = np.array([
[0, 1, 1, 0],
[0, 0, 1, 0],
[0, 0, 0, 1],
[0, 0, 0, 0]
])
matrixB = np.array([
[0, 1, 1, 1, 0],
[0, 0, 0, 0, 1],
[0, 0, 0, 1, 0],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 0],
])
# Generate cytoscape network obejct from ndarray
subA = cy.network.create_from_ndarray(matrixA, name='SubA', collection='Collection')
subB = cy.network.create_from_ndarray(matrixB, name='SubB', collection='Collection')
collectionSUID = requests.get(REST_ENDPOINT + "/v1/collections").json()[0]
subASUID = subA._CyNetwork__id
subBSUID = subB._CyNetwork__id
print("Collection with SUID %s" % collectionSUID)
print("SubA with SUID %s" % subASUID)
print("SubB with SUID %s" % subBSUID)
print("Applying layout...")
cy.layout.apply(network=subA)
cy.layout.apply(network=subB)
print("Networks created")
network_states = {
collectionSUID: {},
subASUID: {},
subBSUID: {},
}
node_data = {
subASUID: {},
subBSUID: {}
}
print("Adding to SubA {author: BrettA, description: descA}")
subAData = {"author": "BrettA", "description": "descA"}
respA = addNetworkAttributes(subASUID, subAData)
network_states[subASUID].update(subAData)
print("Adding to SubB {author: BrettB, description: descB}")
subBData = {"author": "BrettB", "description": "descB"}
respB = addNetworkAttributes(subBSUID, subBData)
network_states[subBSUID].update(subBData)
print("Adding extra column to subA nodes")
tableA = subA.get_node_table()
extra = ['a', 'b', 'c', 'd']
tableA['extra'] = extra
node_data[subASUID] = {'extra': extra}
subA.update_node_table(tableA, network_key_col='name', data_key_col='name')
print("Adding to Collection {author: Brett Coll, disease: example}")
collData = {'author': 'Brett Coll', 'disease': 'example'}
resp = addCollectionAttributes(collectionSUID, collData)
network_states[collectionSUID].update(collData)
In [12]:
# Testing basic CyNDEx2 endpoints
resp = requests.get(REST_ENDPOINT + "/cyndex2/v1")
data=resp.json()['data']
assert data['apiVersion'] == "1", "This test is for App Version 1"
print("CyNDEx Version " + data['appVersion'])
print("Getting current network data")
data = getCurrentNetworks()
print("Checking collection attributes")
rootInfo = data['currentRootNetwork']
assert rootInfo['suid'] == collectionSUID, "currentRootNetwork is incorrect"
assert rootInfo['name'] == 'Collection', "Root name incorrect"
assert rootInfo['props']['disease'] == 'example', 'collection attribute "disease" was not added'
assert rootInfo['props']['author'] == 'Brett Coll', 'collection attribute "author" was not added'
print("Checking network attributes")
nets = data['members']
for net in nets:
data = network_states[net['suid']]
if not assertState(data, net['props']):
raise Exception("Attributes {} of network with SUID {} should be in {}".format(data, net['suid'], net['props']))
print("Ready to test...")
In [13]:
uuid = "fd15a70a-c7b5-11e4-951c-000c29cb28fb"
importNetwork(uuid)
In [8]:
print("Testing save collection private")
data = {
"metadata": {
'name': 'private Collection',
},
"isPublic": 'false',
"writeCollection": 'true'
}
data.update(ndex_accounts['public'])
privateUUID = uploadNetwork(subASUID, data)
ndexUrl = ndex_accounts['public']['serverUrl']
resp = requests.get(ndexUrl + '/network/{}/aspect/networkAttributes'.format(privateUUID))
code = resp.json()['errorCode']
assert code == 'NDEx_Unauthorized_Operation_Exception', 'Private collection should unauthorized exception'
print("Check your NDEx account to see that the private Collection network is indeed private with disease='example'")
In [7]:
print("Importing network " + privateUUID)
newCollectionSUID, newPrivateUUID = importNetwork(privateUUID, ndex_accounts['public'])
time.sleep(1)
print("Adding attribute 'extra' to collection")
collData = {'extra': 'updated', 'disease': 'new'}
resp = addCollectionAttributes(newCollectionSUID, collData)
network_states[newCollectionSUID] = network_states[collectionSUID].copy()
network_states[newCollectionSUID].update(collData)
subs = requests.get(REST_ENDPOINT + '/v1/collections/{}/subnetworks'.format(newCollectionSUID)).json()
subSUID = subs[0]
print("Updating collection on NDEx")
data = {'writeCollection': "true", "metadata" : {"name": "collection updated"}, "isPublic": "false"}
data.update(ndex_accounts['public'])
updateUUID = uploadNetwork(subSUID, data, method='put')
assert updateUUID == newPrivateUUID, "Updated collection has a different UUID {} != {}".format(updateUUID, newPrivateUUID)
print("Check your account again, and see that the collection is now public with disease='new'")
# updating public back to private doesn't seem to work
print("Cleanup... removing imported collection")
#for sub in subs:
# requests.delete(REST_ENDPOINT + "/v1/networks/{}".format(sub))
In [ ]:
print("Testing save collection public")
data = {
"metadata": {
'name': 'public Collection',
},
"isPublic": 'true',
"writeCollection": 'true'
}
data.update(ndex_accounts['public'])
publicUUID = uploadNetwork(subASUID, data)
resp = requests.get('http://ndexbio.org/v2/network/{}/aspect/networkAttributes'.format(publicUUID))
data = resp.json()
data = {d['n']: d['v'] for d in data}
if not assertState(network_states[collectionSUID], data):
print("Collection summary on NDEx missing attributes.\n{} not in {}".format(network_states[collectionSUID], data))
In [ ]:
print("Testing update collection public")
In [9]:
print("Testing save network private")
data = {
"metadata": {
'name': 'private SubA',
},
"isPublic": False,
"writeCollection": False
}
data.update(ndex_accounts['public'])
ndexUrl = data['serverUrl']
privateUUID = uploadNetwork(subASUID, data)
resp = requests.get(ndexUrl + '/network/{}/aspect'.format(privateUUID))
code = resp.json()['errorCode']
assert code == 'NDEx_Unauthorized_Operation_Exception', 'Private network should return unauthorized exception'
'''
resp = requests.delete(REST_ENDPOINT + "/v1/networks/{}".format(subASUID))
print(resp.content)
print("Testing import private network")
data = {
'uuid': privateUUID,
}
data.update(ndex_accounts['public'])
print(data)
resp = requests.post(REST_ENDPOINT + '/cyndex2/v1/networks', data=json.dumps(data), headers={'Content-Type': 'application/json'})
print(resp.content)
'''
Out[9]:
In [28]:
print("Testing update private network")
addNetworkAttributes(subASUID, {'extra': "value"})
data = {
"metadata": {
'name': 'private SubA Updated'
},
"isPublic": "false",
"writeCollection": "false"
}
data.update(ndex_accounts['public'])
print(data)
privateUUID2 = uploadNetwork(subASUID, data, method='put')
assert privateUUID == privateUUID2, "UUIDs do not match, {} != {}".format(privateUUID, privateUUID2)
In [ ]:
print("Testing save network public")
data = {
"metadata": {
'name': 'public SubA',
},
"isPublic": 'true',
"writeCollection": 'false'
}
data.update(ndex_accounts['public'])
publicUUID = uploadNetwork(subASUID, data)
resp = requests.get('http://ndexbio.org/v2/network/{}/aspect/networkAttributes'.format(publicUUID))
data = resp.json()
data = {d['n']: d['v'] for d in data}
if not assertState(network_states[subASUID], data):
print("SubA summary on NDEx missing attributes.\n{} not in {}".format(network_states[subASUID], data))
In [ ]:
print("Testing import singleton")
In [ ]:
print("Testing import collection")
In [ ]:
Get SubA with NDEx endpoint and compare nodes, edges, and attributes Add "author" attribute to SubA Update SubA to NDEx via CyNDEx2 Get SubA with NDEx endpoint and verify author Import SubA to cytoscape and compare nodes, edges, and attributes
Push Collection to NDEx via CyNDEx2 Get Collection with NDEx endpoint and verify attributes (collection and subnetworks) Add "author" attribute to Collection Update Collection to NDEx via CyNDEx2 Get Collection with NDEx endpoint and verify author in Collection Add "author" attribute to SubB Update Collection to NDEx via CyNDEx2 Get Collection with NDEx endpoint and verify author in SubB Import Collection to cytoscape and compare nodes, edges, and attributes (collection and subnetworks)
Clear all networks Pull singleton from ndexbutler networks via CyNDEx2 Verify information (attributes, nodeCount) against NDEx endpoint Add/Change author, description attributes E: Try to update, verify error message Upload to NDEx as new network Get net network with NDEx endpoint and compare new attributes Remove from cytoscape Import network again Change attributes again Update on NDEx Verify changes with NDEx endpoint
ISSUES TO CHECK FOR Removing attributes (how to handle null values)
In [ ]:
# Test with large network, ~3.2M edges
requests.get(REST_ENDPOINT + "/cyndex2/v1/networks")