In [1]:
from __future__ import print_function
import os,sys
import requests
from bs4 import BeautifulSoup
import json, string, getpass
try:
input = raw_input
except NameError: # Python 3
pass
In [2]:
# get VAMPS username and password
vamps_username = input("Enter your VAMPS username: ")
vamps_password = getpass.getpass("Enter your VAMPS password: ")
if vamps_username == 'guest':
raise ValueError('please register instead of using guest account')
# vamps: https://vamps2.mbl.edu
# vampsdev (private) http://vampsdev.jbpc-np.mbl.edu:8124
# localhost: http://localhost:3000
VAMPS_HOST = 'https://vamps2.mbl.edu'
vamps_session = requests.Session()
def vamps_post(url_suffix, data={}, timeout=15):
url = VAMPS_HOST + url_suffix
return vamps_session.post(url, data=data, timeout=timeout)
# attempt login to VAMPS
r = vamps_post('/users/login', data={
'username': vamps_username,
'password': vamps_password
})
# report login status
if r.url == VAMPS_HOST + '/users/login':
raise ValueError('Login not successful')
elif r.url == VAMPS_HOST + '/':
print('Login successful')
else:
raise ValueError('unexpected server response')
In [3]:
def input_yn(msg):
msg += ' ("Y" or "N"): '
resp = input(msg)
return resp.lower()[:1] == 'y'
upload = input_yn("Do you want to use an already existing config file?")
# to upload config:
if upload:
file = input('Enter JSON Config File: ')
with open(file) as f:
config = json.load(f)
id_list = False
else:
id_list = input_yn("Do you want to search through datasets or see all you have access to?")
In [4]:
if id_list:
search = input("Enter dataset you are looking for to get a list of matches: ")
data = {
'search_string': search, # If not empty will search for projects with string in
# project name, title or description (case insensitive)
# Uncomment below line to include project information
#'include_info':'' # if present, data will include project information
}
r = vamps_post('/api/find_user_projects', data)
result = r.json()
print(result)
In [5]:
if not upload:
# default config (if not uploaded); Emily B.'s MVCO eukaryote data
config = {
"api":"1",
"source":"VAMPS-API",
"update_data":1,
"normalization":"none", # none, maximum, frequency
"selected_distance":"morisita-horn", # morisita-horn, jaccard, kulczynski, canberra bray-curtis
"tax_depth":"family", # domain, phylum, klass, order, family, genus, species, strain
"domains":["Eukarya"], #["Archaea","Bacteria","Eukarya","Organelle","Unknown"]
"include_nas":"yes", # yes or no
"min_range":0, # integer 0-99
"max_range":100, # integer 1-100
# Must be a valid project - with correct permissions for the above user.
# Default is Emily B.'s MVCO eukaryote data
'project':'MVCO_ciliate_timeseries2',
# Currently avalable: "dheatmap", "piecharts", "barcharts", "counts_matrix", "metadata_csv", "adiversity", "fheatmap", "dendrogram"
# Default is Alpha Diversity visualization
'image':'piecharts'
}
In [6]:
# get project ids:
r = vamps_post('/api/get_dids_from_project', data=config)
config['ds_order'] = r.json()
print(config['ds_order'])
In [7]:
# Get timestamp to be used as a prefix for files:
r = vamps_post('/visuals/view_selection', data=config)
soup = BeautifulSoup(r.text, "lxml") # html5lib lxml html.parser
ts = soup.find(id="ts_for_bs").string
print("Timestamp/file prefix:",ts)
In [8]:
def vamps_get_file(filename, local_file=None):
if local_file is None:
local_file = filename
url = VAMPS_HOST + '/' + filename
r = requests.get(url, stream=True)
r.raise_for_status()
with open(local_file, 'wb') as fout:
for block in r.iter_content(1024):
fout.write(block)
biom_matrix_file = ts+'_count_matrix.biom'
vamps_get_file(biom_matrix_file)
In [9]:
r = vamps_post('/api/create_image', data=config, timeout=30)
try:
result = r.json()
except:
raise ValueError(r.text())
local_filename = result['filename']
return_result = result['html']
print(local_filename)
vamps_get_file(local_filename)
print('Done writing local file:',local_filename)
In [10]:
from IPython.core.display import display, HTML
def display_viz(html):
style = "<style>.container { width:100% !important; }</style>"
return HTML(style + html)
display_viz(return_result)
Out[10]:
In [11]:
data = {"project": config['project']}
r = vamps_post('/api/get_metadata_from_project', timeout=15, data=data)
result = r.json()
print("Loaded metadata")
In [12]:
lat = 0
lon = 0
# get lat/lon
for ids in result:
for mdname in result[ids]:
if mdname == "longitude":
lon = float(result[ids][mdname])
elif mdname == "latitude":
lat = float(result[ids][mdname])
def show_example_metadata(result, n=3):
for dsid in list(result)[:n]:
md = result[dsid]
for mdname in md:
print('{}: {}'.format(mdname, md[mdname]))
print()
show_example_metadata(result)
In [13]:
half_size = 0.1
bounding_box = {
'nw_lat': lat + half_size if lat > 0 else lat - half_size,
'nw_lon': lon + half_size if lon > 0 else lon - half_size,
'se_lat': lat - half_size if lat > 0 else lat + half_size,
'se_lon': lon - half_size if lon > 0 else lon + half_size,
}
found = False
data = bounding_box
r = vamps_post('/api/find_projects_in_geo_area', timeout=15, data=data)
result = r.json()
for sets in result:
if sets == 'AFP_MVCO_Bv6':
print("Found second project dataset using latitude/longitude data")
# if MVCO bacteria data is found, change config project
config['project'] = sets
found = True
break
In [14]:
# change config file project to second project if not found using lat/lon metadata
if not found:
config['project'] = 'AFP_MVCO_Bv6'
config['domains'] = ["Bacteria"]
config["tax_depth"] = "family"
In [15]:
# get project ids:
r = vamps_post('/api/get_dids_from_project', timeout=15, data=config)
config['ds_order'] = r.json()
# exclude 4 datasets
if config['project'] == 'AFP_MVCO_Bv6':
exclude = [336408, 336409, 336407, 336410]
config['ds_order'] = [dsid for dsid in config['ds_order'] if dsid not in exclude]
In [16]:
# Get timestamp (filename prefix):
r = vamps_post('/visuals/view_selection', timeout=15, data=config)
soup = BeautifulSoup(r.text, "lxml") # html5lib lxml html.parser
ts = soup.find(id="ts_for_bs").string
print("Timestamp/file prefix:",ts)
In [17]:
biom_matrix_file = ts+'_count_matrix.biom'
vamps_get_file(biom_matrix_file)
In [18]:
r = vamps_post('/api/create_image', timeout=30, data=config)
try:
result = r.json()
except:
raise ValueError(r.text)
local_filename = result['filename']
return_result = result['html']
print(local_filename)
vamps_get_file(local_filename)
print('Done writing local file: {}'.format(local_filename))
In [19]:
display_viz(return_result)
Out[19]:
In [20]:
data = {"project": config['project']}
r = vamps_post('/api/get_metadata_from_project', timeout=15, data=data)
result = r.json()
print("Loaded metadata")
In [21]:
show_example_metadata(result)