In [1]:

    

import sys
# import os
sys.path.insert(0,'/global/homes/b/bpb/repos/metatlas')
sys.path.insert(1,'/global/homes/b/bpb/repos/labkey-api-python/')
# from IPython.core.display import display, HTML
# display(HTML("<style>.container { width:100% !important; }</style>"))
from metatlas.helpers import feature_tools as ft 
import multiprocessing as mp
import pandas as pd
import numpy as np

from sklearn import linear_model, datasets
from sklearn.linear_model import HuberRegressor
import matplotlib.pyplot as plt
from matplotlib import cm

import labkey as lk
from matplotlib import pyplot as plt
import uuid #this is because generated lists in labkey assume "name" is primary key and must be unique
%matplotlib notebook


    

In [2]:

    

def make_tasks(ref_df):
    """
    extract from what the labkey server returns metatlas formated values
    
    from the query, paths to files are returned and reference signals are returned
    
    Args:
        ref_df (pandas dataframe): the rows returned from lims system
    
    Returns:
        processing_inputs (list of dicts): 
    """
    processing_inputs = {}
    #todo: add in compound adduct, etc if its given.  currently will delete it. get an exhaustive list and do an "if in"
    #if you delete them here, they won't get written to the hdf5 file and will be lost in the process
    atlas_cols = ['label','mz','polarity','rt_max','rt_min','rt_peak']
    for ref_mix in ref_df['reference_mix'].unique():
        atlas = ref_df[ref_df['reference_mix']==ref_mix].drop_duplicates(['label','polarity'])[atlas_cols]
        files = ref_df[ref_df['reference_mix']==ref_mix].drop_duplicates(['hdf5_file','polarity'])
        processing_inputs[ref_mix] = {}
        processing_inputs[ref_mix]['atlas_pos'] = atlas[atlas['polarity']=='positive']
        processing_inputs[ref_mix]['atlas_neg'] = atlas[atlas['polarity']=='negative']          
        processing_inputs[ref_mix]['files_pos'] = files[files['polarity']=='positive']['hdf5_file'].tolist()
        processing_inputs[ref_mix]['files_neg'] = files[files['polarity']=='negative']['hdf5_file'].tolist()
    return processing_inputs



def get_control_tasks(control_type='qc'):
    """
    Query to get lcmsruns and reference signals from lims system
    
    This function gets the lcmsruns and reference signals by an 
    inner join on the reference_mix identifed for the lcmsrun run
    and the ReferenceMZRT.
    
    Args:
        control_type (str):'qc' or 'istd'
        
    Returns:
        df (pandas dataframe): this will contain the lcmsruns and reference signals to hunt for
    
    """
    
    sql = """
    SELECT run.date_finished,
    run.hdf5_file,
    run.mass_spectrometry,
    refs.mz,
    refs.rt_min,
    refs.rt_max,
    refs.rt_peak,
    refs.polarity,
    refs.label,
    refs.reference_mix
    FROM "/Project Tracking/".samples.lcmsrun_sample as run
    INNER JOIN "/Metabolite Atlas/".lists.ReferenceMZRT as refs 
    ON run.%s_reference_mix = refs.reference_mix
    AND run.liquid_chromatography=refs.liquid_chromatography
    """%control_type
    
    
    labkey_server = 'metabolomics.app.labkey.host'

#     project_name = 'Metabolite Atlas'  # Project folder name
#     m_con = lk.utils.create_server_context(labkey_server, project_name, use_ssl=True)

    project_name = 'Project Tracking'  # Project folder name
    pt_con = lk.utils.create_server_context(labkey_server, project_name, use_ssl=True)

    sql_result = lk.query.execute_sql(pt_con, 'lists', sql)
    if sql_result is not None:
        print("execute_sql: There are " + str(sql_result['rowCount']) + " rows.")
    else:
        print('execute_sql: Failed to load results from ' + schema + '.' + table)
    
    df = pd.DataFrame(sql_result['rows'])
    
    #TODO: This logic must be codable in the SQL query above
    pos_idx = (df['mass_spectrometry'].str.contains('_POS')) & (df['polarity']=='positive')
    neg_idx = (df['mass_spectrometry'].str.contains('_NEG')) & (df['polarity']=='negative')
    fps_idx = df['mass_spectrometry'].str.contains('_FPS')
    df = df[pos_idx|neg_idx|fps_idx]

    return df


def get_reference_mzrt(reference_type='EMA_HILICz-v11'):
    """
    Query to get reference signals from lims system
    
    Args:
        reference_mix (str):'EMA_HILICz-v11', 'polar internal standards v5', or 'QCsopV3'
        
    Returns:
        df (pandas dataframe): this will contain the reference signals to hunt for
    
    """
    
    sql = """
    SELECT refs.inchi_key,
    refs.mz,
    refs.rt_min,
    refs.rt_max,
    refs.rt_peak,
    refs.polarity,
    refs.adduct,
    refs.label,
    refs.reference_rt_space,
    refs.liquid_chromatography,
    refs.mass_spectrometry,
    refs.reference_mix
    FROM "/Metabolite Atlas/".lists.ReferenceMZRT as refs
    WHERE refs.reference_mix = '%s'
    """%reference_type
    
    labkey_server = 'metabolomics.app.labkey.host'

    project_name = 'Project Tracking'  # Project folder name
    pt_con = lk.utils.create_server_context(labkey_server, project_name, use_ssl=True)

    sql_result = lk.query.execute_sql(pt_con, 'lists', sql)
    if sql_result is not None:
        print("execute_sql: There are " + str(sql_result['rowCount']) + " rows.")
    else:
        print('execute_sql: Failed to load results from ' + schema + '.' + table)
    
    df = pd.DataFrame(sql_result['rows'])
    
#     #TODO: This logic must be codable in the SQL query above
#     pos_idx = (df['mass_spectrometry'].str.contains('_POS')) & (df['polarity']=='positive')
#     neg_idx = (df['mass_spectrometry'].str.contains('_NEG')) & (df['polarity']=='negative')
#     fps_idx = df['mass_spectrometry'].str.contains('_FPS')
#     df = df[pos_idx|neg_idx|fps_idx]

    return df

def get_lcmsruns(filename='%'):
    """
    Query to get reference signals from lims system
    
    Args:
        filename (str): lcmsruns will be filtered where name contains string. 
        Use wildcards '%string%'.  Use '%' for unfiltered.
        
    Returns:
        df (pandas dataframe): this will contain the reference signals to hunt for
    
    """
    
    sql = """
    SELECT run.date_finished,
    run.hdf5_file,
    run.mass_spectrometry,
    run.name,
    run.coefficient,
    run.intercept,
    run.mean_squared_error
    --run.reference_rt_space
    FROM "/Project Tracking/".samples.lcmsrun_sample as run
    WHERE run.name LIKE '%s'
    """%filename
    
#     TODO TODO TODO
    #     TODO TODO TODO
#  ONCE EXPERIMENTS ARE STORED, LOOKUP AND GRAB THE EXPERIMENT NAME.  CURRENTLY, EXTRAT FROM HDF5 PATH!!!!!!!!
#     TODO TODO TODO
#     TODO TODO TODO
#     TODO TODO TODO
    
    labkey_server = 'metabolomics.app.labkey.host'

    project_name = 'Project Tracking'  # Project folder name
    pt_con = lk.utils.create_server_context(labkey_server, project_name, use_ssl=True)

    sql_result = lk.query.execute_sql(pt_con, 'lists', sql)
    if sql_result is not None:
        print("execute_sql: There are " + str(sql_result['rowCount']) + " rows.")
    else:
        print('execute_sql: Failed to load results from ' + schema + '.' + table)
    
    df = pd.DataFrame(sql_result['rows'])
    df['date_finished'] = pd.to_datetime(df['date_finished'])
    
    df['experiment'] = df['hdf5_file'].apply(lambda x: x.split('/')[-2] if x is not None else x)
#     #TODO: This logic must be codable in the SQL query above
#     pos_idx = (df['mass_spectrometry'].str.contains('_POS')) & (df['polarity']=='positive')
#     neg_idx = (df['mass_spectrometry'].str.contains('_NEG')) & (df['polarity']=='negative')
#     fps_idx = df['mass_spectrometry'].str.contains('_FPS')
#     df = df[pos_idx|neg_idx|fps_idx]

    return df


    

In [ ]:

    

labkey_server = 'metabolomics.app.labkey.host'

project_name = 'Metabolite Atlas'  # Project folder name
pt_con = lk.utils.create_server_context(labkey_server, project_name, use_ssl=True)

sql_result = lk.query.execute_sql(, 'lists', sql)
if sql_result is not None:
    print("execute_sql: There are " + str(sql_result['rowCount']) + " rows.")
else:
    print('execute_sql: Failed to load results from ' + schema + '.' + table)

df = pd.DataFrame(sql_result['rows'])


    

In [3]:

    

df = get_control_tasks(control_type='qc')
df.head()


    

    





ServerContextErrorTraceback (most recent call last)
<ipython-input-3-b7d3e3401767> in <module>()
----> 1 df = get_control_tasks(control_type='qc')
      2 df.head()

<ipython-input-2-2c958eb8ad39> in get_control_tasks(control_type)
     71     pt_con = lk.utils.create_server_context(labkey_server, project_name, use_ssl=True)
     72 
---> 73     sql_result = lk.query.execute_sql(pt_con, 'lists', sql)
     74     if sql_result is not None:
     75         print("execute_sql: There are " + str(sql_result['rowCount']) + " rows.")

/global/homes/b/bpb/repos/labkey-api-python/labkey/query.pyc in execute_sql(server_context, schema_name, sql, container_path, max_rows, sort, offset, container_filter, save_in_session, parameters, required_version, timeout)
    141         payload['apiVersion'] = required_version
    142 
--> 143     return server_context.make_request(url, payload, timeout=timeout)
    144 
    145 

/global/homes/b/bpb/repos/labkey-api-python/labkey/utils.pyc in make_request(self, url, payload, headers, timeout, method, non_json_response, file_payload)
    109             return handle_response(raw_response, non_json_response)
    110         except RequestException as e:
--> 111             handle_request_exception(e, server_context=self)
    112 
    113 

/global/homes/b/bpb/repos/labkey-api-python/labkey/utils.pyc in handle_request_exception(e, server_context)
    155     if type(e) in [RequestAuthorizationError, QueryNotFoundError, ServerNotFoundError]:
    156         raise e
--> 157     raise ServerContextError(server_context, e)
    158 
    159 

ServerContextError: "'500: Error on line 11: Query or table not found: /Project Tracking/.samples.lcmsrun_sample'"

In [4]:

    

def setup_dataframe(input_data,min_intensity=1e5,use_data=False):
    """
    
    """
    if use_data==True:
        df = input_data['ms1_summary'].copy()
        df.set_index('label',drop=True,inplace=True)
    else:
        df = pd.read_hdf(input_data['outfile'],'ms1_summary').set_index('label',drop=True)
    
    if use_data==True:
        atlas = input_data['atlas'].copy()
        atlas.set_index('label',drop=True,inplace=True)
    else:
        atlas = pd.read_hdf(input_data['outfile'],'atlas').set_index('label',drop=True)
    df = pd.merge(df,atlas,how='outer',left_index=True,right_index=True,suffixes=['_measured','_atlas'])
    
    # replace nan with zero for hits with no signal
    df.loc[pd.isna(df['num_datapoints']),['num_datapoints','peak_height','peak_area']] = 0
    
    if use_data==True:
        eic = input_data['ms1_data']
    else:
        eic = pd.read_hdf(input_data['outfile'],'ms1_data')
    eic = eic[eic['i']>min_intensity]
    eic = ft.group_duplicates(eic[['label','rt','mz','i','in_feature']],'label',make_string=False).set_index('label',drop=True)
    eic.columns = ['eic_%s'%c for c in eic.columns]
    df = pd.merge(df,eic,how='outer',left_index=True,right_index=True)
    
    if use_data==True:
        msms = input_data['ms2_data']
    else:
        msms = pd.read_hdf(input_data['outfile'],'ms2_data')
    msms = msms[msms['i']>min_intensity]
    msms = ft.group_duplicates(msms[['label','mz','i','rt','in_feature']],'label',make_string=False).set_index('label',drop=True)
    msms.columns = ['msms_%s'%c for c in msms.columns]
    df = pd.merge(df,msms,left_index=True,right_index=True,how='left')
    
    return df


    

In [5]:

    

ft = reload(ft)
for control_type in ['qc','istd']:
    df = get_control_tasks(control_type=control_type)
    processing_inputs = make_tasks(df)

    for polarity in ['positive','negative']:

        polarity_short = polarity[:3] #pos/neg is nomenclature in the hdf5 files

        # define the name of the reference mix you are using
        for reference_mix in processing_inputs.keys(): # 'polar internal standards v5' is typically the only one
            print('working on %s %s'%(polarity,reference_mix))
            atlas = processing_inputs[reference_mix]['atlas_%s'%polarity_short]
#             atlas.reset_index(inplace=True)
            filenames = processing_inputs[reference_mix]['files_%s'%polarity_short]           
            input_data = ft.setup_file_slicing_parameters(atlas,filenames,polarity=polarity,project_dir=False)

            print('getting data')
            print('making eics and msms.')
            all_dfs = []
            for f in input_data:
                data = ft.get_data(f,return_data=True,save_file=False)
                df = setup_dataframe(data,use_data=True)
                #TODO!!!! THIS MUST BE PRESERVED AS THE NAME FIELD (PK) FOR LCMSRUN_SAMPLES !!!!!!!
                df['lcmsrun'] = '%s.mzML'%os.path.basename(f['lcmsrun'].split('.')[0])
                df['hdf5_file'] = f['lcmsrun']
                df['feature_file'] = f['outfile']
                df['polarity'] = f['polarity']
                df['reference_mix'] = reference_mix
                df.reset_index(inplace=True)
                #basic cleanup
                df.loc[pd.isna(df['num_datapoints']),'num_datapoints'] = 0
                #TODO: Decide what to do about uuids
                df['name'] = [str(uuid.uuid4()) for i in range(df.shape[0])]

                #string arrays to fixed precision
                print('stringifying ndarrays')
                str_df = df.copy()
                d = df.to_dict('records')
                for k,v in d[0].items():
                    if any('ndarray' in str(type(dd[k])) for dd in d):
                        # convert np.array to string if nan==float then None
                        # convert np.array to string if nan==float then None
                        if 'mz' in k:
                            p=5 #gets you better than 1 ppm
                        elif 'rt' in k:
                            p=3 #60 milliseconds
                        else:
                            p=1 #intensity becomes low precision


                        if 'in_feature' in k:
                            t = 'bool'
                        else:
                            t = 'float'
                        np.set_printoptions(suppress=True,formatter={'float_kind':('{:0.%df}'%p).format},threshold=1e6) 
                        str_df[k] = str_df[k].apply(lambda x: np.array2string(x.astype(t),precision=p,floatmode='fixed',separator=',',max_line_width=1e10) if type(x)!=float else None)
                    
                #send to lims
                print('sending to lims system')
                labkey_server = 'metabolomics.app.labkey.host'
                project_name = 'Project Tracking'  # Project folder name
                pt_con = lk.utils.create_server_context(labkey_server, project_name, use_ssl=True)        
                insertResult = lk.query.insert_rows(pt_con, 'lists', '%s_signals'%control_type, str_df.to_dict('records'))
                print('done\n')


#             print('exporting csv')
#             df.to_csv('/global/homes/b/bpb/Downloads/%s_signals_%s_%s.csv'%(control_type,reference_mix,polarity_short),index=None)
            
    

    
# ft = reload(ft)
# d = ft.get_data(input_data[1],return_data=True,save_file=False)

# ms1_summary = d['ms1_summary']

# df = setup_dataframe(input_data[0],reference_mix)
# df.head()


    

    




execute_sql: There are 6072 rows.
working on positive QCsopV3
getting data
making eics and msms.
stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

working on negative QCsopV3
getting data
making eics and msms.
stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

execute_sql: There are 31680 rows.
working on positive polar internal standards v5
getting data
making eics and msms.
stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

working on negative polar internal standards v5
getting data
making eics and msms.
stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

stringifying ndarrays
sending to lims system
done

In [ ]:

    

# ft = reload(ft)
# for control_type in ['qc','istd']:
#     df = get_control_tasks(control_type=control_type)
#     processing_inputs = make_tasks(df)

#     for polarity in ['positive','negative']:

#         polarity_short = polarity[:3] #pos/neg is nomenclature in the hdf5 files

#         # define the name of the reference mix you are using
#         for reference_mix in processing_inputs.keys(): # 'polar internal standards v5' is typically the only one
#             print('working on %s %s'%(polarity,reference_mix))
#             atlas = processing_inputs[reference_mix]['atlas_%s'%polarity_short]
#             atlas.reset_index(inplace=True)
#             filenames = processing_inputs[reference_mix]['files_%s'%polarity_short]
            
#             input_data = ft.setup_file_slicing_parameters(atlas,filenames,polarity=polarity,project_dir='test_labkey')

#             print('getting data')
# #             pool = mp.Pool(processes=8)
# #             data = pool.map(ft.get_data,input_data)
# #             pool.close()
# #             pool.join()
            
#             print('making eics and msms.')
#             all_dfs = []
#             for f in input_data:
#                 data = ft.get_data(f,return_data=False,save_file=True)
#                 df = setup_dataframe(f,use_data=False)
#                     #TODO!!!! THIS MUST BE PRESERVED AS THE NAME FIELD (PK) FOR LCMSRUN_SAMPLES !!!!!!!
#                 df['lcmsrun'] = '%s.mzML'%os.path.basename(f['lcmsrun'].split('.')[0])
#                 df['hdf5_file'] = f['lcmsrun']
#                 df['feature_file'] = f['outfile']
#                 df['polarity'] = f['polarity']
#                 df['reference_mix'] = reference_mix
#                 all_dfs.append(df)
#             df = pd.concat(all_dfs)
#             df.reset_index(inplace=True)
            
#             #basic cleanup
#             df.loc[pd.isna(df['num_datapoints']),'num_datapoints'] = 0

#             df['name'] = [str(uuid.uuid4()) for i in range(df.shape[0])]

#             #string arrays to fixed precision
#             print('stringifying ndarrays')
#             str_df = df.copy()
#             d = df.to_dict('records')
#             for k,v in d[0].items():
#                 if any('ndarray' in str(type(dd[k])) for dd in d):
#                     # convert np.array to string if nan==float then None
#                     # convert np.array to string if nan==float then None
#                     if 'mz' in k:
#                         p=5 #gets you better than 1 ppm
#                     elif 'rt' in k:
#                         p=3 #60 milliseconds
#                     else:
#                         p=1 #intensity becomes low precision
                        

#                     if 'in_feature' in k:
#                         t = 'bool'
#                     else:
#                         t = 'float'
#                     np.set_printoptions(suppress=True,formatter={'float_kind':('{:0.%df}'%p).format},threshold=1e6) 
#                     str_df[k] = str_df[k].apply(lambda x: np.array2string(x.astype(t),precision=p,floatmode='fixed',separator=',',max_line_width=1e10) if type(x)!=float else None)
            
#             print('exporting csv')
#             df.to_csv('/global/homes/b/bpb/Downloads/%s_signals_%s_%s.csv'%(control_type,reference_mix,polarity_short),index=None)
            
#             print('sending to lims system')
#             labkey_server = 'metabolomics.app.labkey.host'
#             project_name = 'Project Tracking'  # Project folder name
#             pt_con = lk.utils.create_server_context(labkey_server, project_name, use_ssl=True)        
#             insertResult = lk.query.insert_rows(pt_con, 'lists', '%s_signals'%control_type, str_df.to_dict('records'))
#             print('done')
    

    
# # ft = reload(ft)
# # d = ft.get_data(input_data[1],return_data=True,save_file=False)

# # ms1_summary = d['ms1_summary']

# # df = setup_dataframe(input_data[0],reference_mix)
# # df.head()


    

In [6]:

    

def get_reference_signals(control_type='qc'):
    """
    Query to get lcmsruns and reference signals from lims system
    
    This function gets the lcmsruns and reference signals by an 
    inner join on the reference_mix identifed for the lcmsrun run
    and the ReferenceMZRT.
    
    Args:
        control_type (str):'qc' or 'istd'
        
    Returns:
        df (pandas dataframe): this will contain the lcmsruns and reference signals to hunt for
    
    """
    
    sql = """
    SELECT refs.peak_height,
    refs.rt_peak_measured,
    refs.rt_peak_atlas,
    refs.mz_centroid,
    refs.mz,
    refs.hdf5_file,
    refs.lcmsrun,
    refs.polarity,
    refs.label,
    refs.reference_mix
    FROM "/Project Tracking/".lists.%s_signals as refs
    """%control_type
    
    
    labkey_server = 'metabolomics.app.labkey.host'

#     project_name = 'Metabolite Atlas'  # Project folder name
#     m_con = lk.utils.create_server_context(labkey_server, project_name, use_ssl=True)

    project_name = 'Project Tracking'  # Project folder name
    pt_con = lk.utils.create_server_context(labkey_server, project_name, use_ssl=True)

    sql_result = lk.query.execute_sql(pt_con, 'lists', sql)
    if sql_result is not None:
        print("execute_sql: There are " + str(sql_result['rowCount']) + " rows.")
    else:
        print('execute_sql: Failed to load results from ' + schema + '.' + table)
    
    df = pd.DataFrame(sql_result['rows'])
    

    return df
df = get_reference_signals(control_type='qc')
df.head()


    

    




execute_sql: There are 5676 rows.






    
Out[6]:







  

    

      

      
_labkeyurl_lcmsrun
      
_labkeyurl_reference_mix
      
hdf5_file
      
label
      
lcmsrun
      
mz
      
mz_centroid
      
peak_height
      
polarity
      
reference_mix
      
rt_peak_atlas
      
rt_peak_measured
    
  
  

    

      
0
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
carnitine; [M]+
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
161.105193
      
NaN
      
0.0
      
positive
      
QCsopV3
      
13.285827
      
NaN
    
    

      
1
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/project/projectdirs/metatlas/raw_data/jsjorda...
      
acetylcholine; [M-2H]-
      
20190118_JJ_KZ_Switchgrass_Greenhouse_Bulk1017...
      
144.103002
      
NaN
      
0.0
      
negative
      
QCsopV3
      
1.955286
      
NaN
    
    

      
2
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
lactic acid; [M+H]+
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
91.038994
      
NaN
      
0.0
      
positive
      
QCsopV3
      
5.064399
      
NaN
    
    

      
3
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/project/projectdirs/metatlas/raw_data/jsjorda...
      
carnitine; [M-2H]-
      
20190111_JJ_KZ_Switchgrass_Greenhouse_Bulk1017...
      
159.090640
      
NaN
      
0.0
      
negative
      
QCsopV3
      
13.285827
      
NaN
    
    

      
4
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
mannitol; [M+H]+
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
183.086338
      
NaN
      
0.0
      
positive
      
QCsopV3
      
9.534507
      
NaN
    
  

In [89]:

    

e = str(e)
e.strip()


    

    
Out[89]:





'Failed to solve the system of normal equations'

In [94]:

    

# fig,ax = plt.subplots(nrows=6,ncols=8,figsize=(14,8))
# ax = ax.flatten()

print('getting reference signals')
df = get_reference_signals(control_type='qc')

print('sending to lims system')
labkey_server = 'metabolomics.app.labkey.host'
project_name = 'Project Tracking'  # Project folder name
pt_con = lk.utils.create_server_context(labkey_server, project_name, use_ssl=True)        
# insertResult = lk.query.insert_rows(pt_con, 'lists', '%s_signals'%control_type, str_df.to_dict('records'))

for i,f in enumerate(df.lcmsrun.unique()):
    small_df = df[df.lcmsrun==f]
    small_df = small_df[small_df['rt_peak_measured']!='NaN']
    small_df = small_df[small_df['peak_height']>1e5]
    small_df.reset_index(drop=True,inplace=True)
    x = small_df.rt_peak_atlas.values.reshape(-1, 1)
    y = small_df.rt_peak_measured.values#.flatten().reshape(-1, 1)
    # Robustly fit linear model with RANSAC algorithm
    ransac = linear_model.RANSACRegressor()
    mse = 1e9
    # do the fit 10 times just in case it fails a few times.  Keep the best one.
    for i in range(10):
        try: #The ransac code failes about 1 in 100 times.
            ransac.fit(x, y)
            y_pred = ransac.predict(x)
            temp_mse = np.mean((y_pred-y)**2)
            if temp_mse < mse:
                m = ransac.estimator_.coef_[0]
                b = ransac.estimator_.intercept_
                mse = temp_mse
        except SystemError as e:
            e = str(e)
            e = e.strip()
            if e != "Failed to solve the system of normal equations":
                raise
            else:
                print("caught!")

        
#     ax[i].plot(x,y,'.')
#     ax[i].plot(line_X,line_y_ransac,'k-')
    new_data = {'Name':f,'coefficient':m,'intercept':b,'mean_squared_error':mse, 'reference_rt_space':small_df.loc[0,'reference_mix']}
    insertResult = lk.query.update_rows(pt_con, 'samples', 'lcmsrun_sample', [new_data])
    
    print(small_df.shape,new_data['coefficient'],new_data['intercept'],new_data['mean_squared_error'])

    
# plt.tight_layout()


    

    




getting reference signals
execute_sql: There are 5676 rows.
sending to lims system
((115, 12), 0.9929835305567092, -0.10319432197409902, 0.04313524296693261)
((113, 12), 0.9840986862491437, -0.16357551510622065, 0.07025551686476593)
((24, 12), 0.991278202099076, -0.08520439291736905, 0.20871262197291104)
((114, 12), 0.9854974040369027, -0.1416802946523614, 0.06360357668997346)
((115, 12), 0.9926450662603816, -0.10687420085351955, 0.044509366682111424)
((113, 12), 0.983703135970729, -0.16040794036122344, 0.07062228279830492)
((52, 12), 0.9827220809929433, -0.052040965583915105, 0.057285087615479444)
((113, 12), 0.9825417287617547, -0.11836952555488985, 0.0666503099036059)
((115, 12), 0.9926166720187832, -0.12674384673717995, 0.04559639636386499)
((114, 12), 0.9842565869047881, -0.13705630413815398, 0.06807205944420963)
((116, 12), 0.985640958006002, -0.14164113266095468, 0.07002012096012725)
((111, 12), 0.9834414058072672, -0.15628992750333046, 0.07165467080026702)
((112, 12), 0.9836657217897071, -0.16772945219620253, 0.07152582990982538)
((115, 12), 0.984644126633879, -0.13748391052668782, 0.06466064024123896)
((113, 12), 0.9854184875875948, -0.12892749986013718, 0.06489353272731395)
((110, 12), 0.9840527740072097, -0.16768595324876365, 0.07203560567557375)
((115, 12), 0.985506588212584, -0.16180556650521133, 0.07104116025667008)
((114, 12), 0.9823934727214259, -0.12174972284752227, 0.0669743616819866)
((115, 12), 0.9846450156498465, -0.13679697269008606, 0.06543278604472672)
((111, 12), 0.9836262745942064, -0.16134335435837038, 0.0709354197110926)
((119, 12), 0.9833333601637892, -0.17650827323840285, 0.0848898714875991)
((115, 12), 0.9854939642872957, -0.1599367662014917, 0.07137844992793545)
((114, 12), 0.9861244035795507, -0.14616819270113582, 0.0618500319592751)
caught!
((114, 12), 0.9824580734260976, -0.13930343729147573, 0.06471158178071064)
((115, 12), 0.9920730735139035, -0.10711092383053557, 0.044502389760568625)
((111, 12), 0.9837154830029416, -0.16092986216026556, 0.07187965173309706)
((62, 12), 0.9852826956087841, -0.17789603493202985, 0.06594117053218362)
((114, 12), 0.9838667006616861, -0.12077130714139726, 0.06411855894227282)
((116, 12), 0.9820636636719743, -0.13227899376466618, 0.0665412346356809)
((114, 12), 0.9848748811266798, -0.13316143150169185, 0.06161040678153633)
((60, 12), 0.9857276678688811, -0.22454699606317957, 0.06616782820833036)
((53, 12), 0.9809226594024547, -0.08691565723816826, 0.06307044292395549)
((51, 12), 0.9803266018228736, -0.10895874753488091, 0.06979274762110878)
((115, 12), 0.9934510949074153, -0.11233192006360072, 0.046250746195351286)
((115, 12), 0.9931791384790687, -0.12389061061933007, 0.04429080571732234)
((115, 12), 0.9938236960259279, -0.11587175010091257, 0.04234640377653712)
((115, 12), 0.9926372412679758, -0.1286543958750947, 0.04530442787272793)
((115, 12), 0.9933627300314847, -0.11672202595573641, 0.04504935385758191)
((115, 12), 0.9932597871366837, -0.12828617072767287, 0.043711353340369345)
((60, 12), 0.9855805364982982, -0.15018688786125245, 0.05450177872149108)
((115, 12), 0.9848626533436309, -0.14757925633066107, 0.06499331643953848)
((115, 12), 0.9931947143507318, -0.11971136394778438, 0.04345836928602386)
((115, 12), 0.9927044031255348, -0.13017901405976282, 0.04534401185587574)
((115, 12), 0.9928607333402233, -0.10754225141883791, 0.044594809684815)
((115, 12), 0.9923455237389701, -0.12001877672316087, 0.04518738080323946)
((114, 12), 0.9920453656833605, -0.12619286825940448, 0.04601213693695351)

In [112]:

    

# fig,ax = plt.subplots(nrows=6,ncols=8,figsize=(14,8))
# ax = ax.flatten()

print('getting reference signals')
df = get_reference_signals(control_type='istd')

print('sending to lims system')
labkey_server = 'metabolomics.app.labkey.host'
project_name = 'Project Tracking'  # Project folder name
pt_con = lk.utils.create_server_context(labkey_server, project_name, use_ssl=True)        
# insertResult = lk.query.insert_rows(pt_con, 'lists', '%s_signals'%control_type, str_df.to_dict('records'))

for i,f in enumerate(df.lcmsrun.unique()):
    small_df = df[df.lcmsrun==f]
    small_df = small_df[small_df['rt_peak_measured']!='NaN']
    small_df = small_df[small_df['peak_height']>1e5]
    small_df.reset_index(drop=True,inplace=True)
    if small_df.shape[0] > 15:
        x = small_df.rt_peak_atlas.values.reshape(-1, 1)
        y = small_df.rt_peak_measured.values#.flatten().reshape(-1, 1)
        # Robustly fit linear model with RANSAC algorithm
        ransac = linear_model.RANSACRegressor()
        mse = 1e9
        # do the fit 10 times just in case it fails a few times.  Keep the best one.
        for i in range(10):
            try: #The ransac code failes about 1 in 100 times.
                ransac.fit(x, y)
                y_pred = ransac.predict(x)
                temp_mse = np.mean((y_pred-y)**2)
                if temp_mse < mse:
                    m = ransac.estimator_.coef_[0]
                    b = ransac.estimator_.intercept_
                    mse = temp_mse
            except SystemError as e:
                e = str(e)
                e = e.strip()
                if e != "Failed to solve the system of normal equations":
                    raise
                else:
                    print("caught!")


    #     ax[i].plot(x,y,'.')
    #     ax[i].plot(line_X,line_y_ransac,'k-')
        new_data = {'Name':f,'coefficient':m,'intercept':b,'mean_squared_error':mse, 'reference_rt_space':small_df.loc[0,'reference_mix']}
        insertResult = lk.query.update_rows(pt_con, 'samples', 'lcmsrun_sample', [new_data])

        print(small_df.shape,new_data['coefficient'],new_data['intercept'],new_data['mean_squared_error'])

    
# plt.tight_layout()


    

    




getting reference signals
execute_sql: There are 16700 rows.
sending to lims system
((19, 12), 1.040196887405894, -0.3628017131950605, 0.04248579094757212)
((19, 12), 1.0420784847726527, -0.39372510720719184, 0.04356235233517654)
((19, 12), 1.0374691225627333, -0.3386692124337563, 0.03174963210487027)
((18, 12), 1.036868359833159, -0.32647357474578115, 0.030149208139303402)
((19, 12), 1.0370860752929187, -0.3306029116409646, 0.028968969148121554)
caught!
((19, 12), 1.036099514253081, -0.3211718613415421, 0.02795907539907442)
((19, 12), 1.0377443284770582, -0.33685993948487447, 0.03524712907096617)
((19, 12), 1.035391742984363, -0.3077176464716549, 0.035267740120121106)
((19, 12), 1.0376146689806167, -0.3631326614003099, 0.018050991302093297)
((18, 12), 1.0436436586451248, -0.4176134176193413, 0.04232958598840843)
((19, 12), 1.0379481755734445, -0.3437389698770318, 0.029279738865981206)
((19, 12), 1.0373151729574435, -0.33737703036738687, 0.0321919162585479)
((19, 12), 1.0369005768915271, -0.3343741050255383, 0.03271922171368295)
((19, 12), 1.0402219692002106, -0.37386483757888217, 0.03723918516234543)
((18, 12), 1.0414154078406732, -0.38759353671935454, 0.03898960973691018)
((19, 12), 1.0362633853002832, -0.33990511816505314, 0.024797901140263)
((19, 12), 1.0402362187386682, -0.37024299544411876, 0.04024923284737267)
((18, 12), 1.0376323862863108, -0.3362182809651636, 0.02888186602038442)
((19, 12), 1.0368633360796606, -0.33370253905127667, 0.029238564228530866)
((19, 12), 1.0385913625334442, -0.3597943710216032, 0.03243652292892294)
((19, 12), 1.0365512432741146, -0.3253704529407097, 0.02830709781781014)
((19, 12), 1.0368280024130103, -0.332051355223485, 0.03162810380358901)
((19, 12), 1.0388954380019548, -0.3638337080729421, 0.03261123643454571)
((19, 12), 1.037071010264518, -0.34144411794552415, 0.029131015438369424)
((19, 12), 1.0328097440311788, -0.29254956566487206, 0.016776370143394777)
((19, 12), 1.0379394353076379, -0.3496141701280078, 0.028698430299557466)
((19, 12), 1.0374859888099255, -0.33926448052599467, 0.031738057557220645)
((19, 12), 1.036256669581944, -0.3482472194371155, 0.025427489843201336)
((19, 12), 1.0371735573793308, -0.3463670631125483, 0.02871564253607616)
((19, 12), 1.03890508137268, -0.36576570002057696, 0.0316239812150929)
((19, 12), 1.0401010019810693, -0.3817956684836234, 0.033662404356243195)
((19, 12), 1.03784526635556, -0.3614952070748563, 0.029387134205455013)
((19, 12), 1.0353505706610513, -0.30773148130445793, 0.028889520229159093)
((19, 12), 1.0407202778387918, -0.3851943644913347, 0.039196612477944745)
((19, 12), 1.0375035317195973, -0.3394674256066658, 0.03201598356694326)
((19, 12), 1.0361898704609085, -0.32199126383716536, 0.02851773246401694)
((19, 12), 1.0374360428732972, -0.34563107151686834, 0.02863944045486735)
((20, 12), 1.0356346880751002, -0.316242449689609, 0.03023134229743188)
((20, 12), 1.035216714752506, -0.31408777401401955, 0.02726138863030189)
((20, 12), 1.0358220213923652, -0.32434745637520773, 0.02717819276684868)
caught!
((20, 12), 1.0375560693426054, -0.3492702625575595, 0.03074838046672938)
((20, 12), 1.0339188005703916, -0.2983184748790977, 0.025929498126656614)
((20, 12), 1.034541198366688, -0.3090715921879877, 0.027037344353462327)
((20, 12), 1.037656881176963, -0.34672370549285136, 0.0318203997908437)
((20, 12), 1.0340440086233083, -0.3001279026796605, 0.027067687611530833)
((20, 12), 1.0363818640660807, -0.3308600865769268, 0.027497417998005987)
((20, 12), 1.0336056847114035, -0.28813053453319437, 0.027146407440672805)
((20, 12), 1.0339786465099385, -0.30044762802034797, 0.027171227699885787)
((20, 12), 1.034602274770004, -0.3059654915581519, 0.027792660079661403)
((20, 12), 1.034699012042046, -0.30519409194571395, 0.02728028777944693)
((20, 12), 1.0355203886784596, -0.3197219108124645, 0.027533553907884544)
((20, 12), 1.0348018969770556, -0.3131617628077611, 0.027168224030294702)
caught!
((20, 12), 1.034960335802752, -0.31248954171827986, 0.026695719517808147)
caught!
((20, 12), 1.0344876010063917, -0.29839398692676783, 0.027445953874436928)
((20, 12), 1.0343381024148404, -0.3005706336980591, 0.028126285867924632)
((20, 12), 1.0348145197149858, -0.30989971956927886, 0.028158985216849904)
((20, 12), 1.0343809472099932, -0.30430192800235534, 0.027480529913391772)
((20, 12), 1.037392632933779, -0.34373184059224193, 0.030547940878023268)
((20, 12), 1.0362549422792173, -0.34873478940541097, 0.02722872905353086)
((20, 12), 1.0392326972071655, -0.3782792508590887, 0.03235459885301292)
((20, 12), 1.0331339425859225, -0.28812896377605035, 0.026691119540321377)
((20, 12), 1.0392511547025574, -0.3767222205954413, 0.03513369678153809)
((20, 12), 1.0354239463560921, -0.3364603937314952, 0.024963605333947984)
((20, 12), 1.0357481090936003, -0.31579418204004395, 0.030480371555302983)
((40, 12), 1.0335084783992505, -0.28117151069500657, 0.026494069877211434)
((20, 12), 1.0357795614963605, -0.32886637027001026, 0.026459249200007605)
((20, 12), 1.034915967798646, -0.3145338812374355, 0.026903003064182814)
((20, 12), 1.0366729778838735, -0.3451876797259486, 0.027151281101990665)
caught!
((20, 12), 1.0335170366706876, -0.2923124286423245, 0.023520892779942685)
((20, 12), 1.0339202839350967, -0.2952187649755031, 0.02808824365089006)
((20, 12), 1.0353161542568428, -0.31690719648134896, 0.027429268081126824)
((20, 12), 1.0386377712481425, -0.3681361944393215, 0.031141666026322297)
((20, 12), 1.034531108310007, -0.31365547596556936, 0.02708075257232152)
((20, 12), 1.035288099957739, -0.3148582796418959, 0.026777730808498316)
((20, 12), 1.0345372155897672, -0.3127129900153254, 0.027100728608821855)
((20, 12), 1.035626203443735, -0.31438633681701245, 0.030405741290971122)
((20, 12), 1.0362380208963498, -0.3292193690279763, 0.028005854182319267)
((20, 12), 1.0364578484710485, -0.33408637800861546, 0.02722129331297981)
((20, 12), 1.033935035605694, -0.2980272157225125, 0.027139846103119303)
((20, 12), 1.0402295985404995, -0.3858771547366424, 0.03710789494444675)
((20, 12), 1.033788514814687, -0.29505629792232596, 0.027599360375486787)
((20, 12), 1.038945622481648, -0.3596788929567728, 0.029752738316338696)
((20, 12), 1.0377198559002982, -0.347089834388259, 0.03434092542365634)
((20, 12), 1.0363056607976524, -0.3357650432353055, 0.027366724306736485)
((20, 12), 1.0322390753064665, -0.27354360081675033, 0.027080593670848863)
((20, 12), 1.0321937936141157, -0.26497755808735013, 0.02733379819293873)
((20, 12), 1.0341812513263775, -0.3042820090047811, 0.026997182764855132)
((20, 12), 1.0374931170466553, -0.34868555549355035, 0.03512986008427843)
caught!
((20, 12), 1.0321145423004845, -0.27119644902746304, 0.027269006470905716)
((20, 12), 1.0362827667862742, -0.3362885560363708, 0.02667528467293682)
((20, 12), 1.0336961954491102, -0.2905529908940578, 0.02642334956977798)
((20, 12), 1.0378952407963742, -0.34910863070680165, 0.03581255241141286)
((20, 12), 1.0339584554853873, -0.3013887644239848, 0.027014458081673232)
((20, 12), 1.039180811480626, -0.370183112803921, 0.03433898497471594)
((20, 12), 1.0366595230725268, -0.33431504388484407, 0.0310880982008212)
((20, 12), 1.0357954172932633, -0.321174318479469, 0.028363443596207655)
((20, 12), 1.0335595760569989, -0.2823628431620855, 0.02720591979076335)
caught!
((20, 12), 1.0358893327550582, -0.33133739413089813, 0.02729504016037574)
((20, 12), 1.034664094834138, -0.3072109459206108, 0.027268421438508928)
((20, 12), 1.0349371999024541, -0.3125019034987671, 0.028475140683687998)
((20, 12), 1.0348449326344762, -0.31449889457921276, 0.027548823973612514)
((20, 12), 1.0357643682252964, -0.3288809010316502, 0.027090516889367998)
((20, 12), 1.0338612032944616, -0.302011787814706, 0.027203290784351474)
((20, 12), 1.030777906777229, -0.24837871463511887, 0.02696380182551289)
((20, 12), 1.037789160139655, -0.36044028297016373, 0.03390970053467873)
((20, 12), 1.0342141214633114, -0.29843528096539185, 0.027566177046994315)
((20, 12), 1.0354790479994143, -0.3223045127211035, 0.0270803862831177)
((20, 12), 1.035543889234138, -0.32868533691028673, 0.027619409546588182)
((20, 12), 1.037186538979344, -0.34321012125303124, 0.027639881641046405)
((20, 12), 1.0346802915133961, -0.2943259689833389, 0.026391962794089025)
((20, 12), 1.0340800905323217, -0.30231550943574975, 0.027328432283018088)
((20, 12), 1.0081166262368866, -0.039764691575169396, 0.06852188403563415)
((40, 12), 1.0346374611986875, -0.300684283248628, 0.029122614587446645)
((20, 12), 1.0327875149373866, -0.2740949489937604, 0.027042712589263045)
((20, 12), 1.0349199152063715, -0.30636645982616806, 0.028068042281999855)
((20, 12), 1.0362273136623343, -0.3205156888553798, 0.03649829112750164)
((20, 12), 1.0347531290744465, -0.3117344655823973, 0.027157222305751738)
((20, 12), 1.0393022314019982, -0.36892215973149983, 0.03650064057394703)
((20, 12), 1.0346570012918246, -0.31164657257982975, 0.02782672510820326)
((20, 12), 1.036130830814212, -0.3310187633319542, 0.027123201942189645)
((20, 12), 1.0334678914003654, -0.2869033807711523, 0.026896762551306706)
((40, 12), 1.0355809434555185, -0.31099845415323735, 0.03136274192383926)
((20, 12), 1.0360340800753012, -0.3282272045622272, 0.02769047409212034)
caught!
((20, 12), 1.03356296167087, -0.2987023058477123, 0.026991018380760668)
((20, 12), 1.035453644666039, -0.33908566116505795, 0.02593381970959806)
((40, 12), 1.0358854626053313, -0.3106348031017372, 0.03174340871300209)
((20, 12), 1.0346676342546768, -0.31070955210183065, 0.027299269313112395)
((20, 12), 1.033758770460794, -0.30020011495653537, 0.02711156933035736)
((20, 12), 1.0350603722197922, -0.3114476759350322, 0.02788954676416506)
((20, 12), 1.0376227782152998, -0.3546351188168826, 0.030869093041916273)
((20, 12), 1.0373649918375412, -0.3467081715082491, 0.032841894997735)
((20, 12), 1.0358348579623926, -0.32234288056237154, 0.03081401984528561)
((40, 12), 1.0344960327943116, -0.29777427381687904, 0.029594428044986354)
((20, 12), 1.0359562338356754, -0.32819042757415384, 0.02746840037389577)
((40, 12), 1.0349996792517702, -0.3027935421694683, 0.03023813122847612)
((20, 12), 1.0369961185097782, -0.3400709725991004, 0.030706212783818243)
((20, 12), 1.0350597846188128, -0.3078514468412428, 0.031131713277497208)
((40, 12), 1.0348765538702527, -0.29950803811760496, 0.02883830210716053)
((20, 12), 1.0380108001600987, -0.35001043397438475, 0.035018090151961785)
((20, 12), 1.0379199336544953, -0.3529786453275798, 0.03557975547620994)
((20, 12), 1.0365075255868461, -0.33283535862395836, 0.030071051720414917)
((20, 12), 1.0335393223087928, -0.28861365601379724, 0.027091138388657364)
((20, 12), 1.0367673487815592, -0.3462580687135142, 0.02682307070654088)
((20, 12), 1.0351068015440896, -0.3321238710136766, 0.026212060048525266)
((40, 12), 1.0357170431494565, -0.3151419962355997, 0.03171756001017246)
((20, 12), 1.0350611596366328, -0.31836212904292527, 0.026940682454706138)
((20, 12), 1.0361997002022054, -0.3242053441495812, 0.029290995470868386)
((40, 12), 1.035299346836383, -0.30201499430655765, 0.0314523215621782)
((20, 12), 1.0360871394634616, -0.33077054353101476, 0.027079334374699133)
((20, 12), 1.0393847357575714, -0.37548939051995406, 0.03555369604984736)
((20, 12), 1.038582588817549, -0.3608168598219413, 0.03110827401054102)
((20, 12), 1.0329117784218234, -0.2803640811940937, 0.027306802459275997)
((20, 12), 1.0380975297656765, -0.36066635861742163, 0.030892175365180553)
((20, 12), 1.0382059484374164, -0.35560908231229504, 0.03406529038747604)
((20, 12), 1.0333121865390966, -0.29169106915139387, 0.027511527606480124)
((20, 12), 1.035764312567144, -0.3199898860925946, 0.0303883674700274)
((20, 12), 1.0336064283722335, -0.27662916136402793, 0.029980333198407445)
((20, 12), 1.0353774921113135, -0.3161022604894172, 0.026885087263290502)
((20, 12), 1.0327693764637418, -0.2837261453933385, 0.027173573789313333)
((20, 12), 1.0341385803347614, -0.2978008971037658, 0.02690895571574133)
((40, 12), 1.0362006758004534, -0.31023867110922476, 0.03281214073220795)
((40, 12), 1.038017759353659, -0.3429389257518682, 0.03393013164160707)
((20, 12), 1.0344373769926927, -0.30745019404193846, 0.026539021988200684)
((20, 12), 1.0352100549036842, -0.3200278048373743, 0.027023339378873423)
((40, 12), 1.037629325821135, -0.3377976925449569, 0.03521271874194088)
caught!
((40, 12), 1.034954548537458, -0.30724255288959695, 0.029676228989409702)
((20, 12), 1.035444892474222, -0.3253136939618496, 0.027654645172249092)
((40, 12), 1.0331643201814051, -0.2723614423398737, 0.029208399909490573)
((20, 12), 1.0358207522317993, -0.3250102472660114, 0.02937386366680015)
((20, 12), 1.0348303801746983, -0.3063537794967124, 0.030230205829921593)
((40, 12), 1.035606652658463, -0.308855799676061, 0.031929900073088754)
((40, 12), 1.0361081466831858, -0.3172424748468494, 0.030354267887128517)
((20, 12), 1.035431086620383, -0.31413696845884803, 0.030536363535326246)
((40, 12), 1.0344003719933417, -0.2921865492157266, 0.03046873015823981)
((20, 12), 1.0360845568056531, -0.33168838059048916, 0.027508098624801088)
((20, 12), 1.0403129655055066, -0.3894933351526312, 0.03536078711057673)
((20, 12), 1.035288887750104, -0.308681268795981, 0.030998768734838734)
((20, 12), 1.0343387404026914, -0.30198069162960106, 0.02757816353268046)
((20, 12), 1.0379138343651555, -0.35936204197244526, 0.03188566304442839)
((20, 12), 1.0366913212638627, -0.3439977208762004, 0.0271120235154172)
((40, 12), 1.0384412412091353, -0.35760360181305384, 0.0350124962059738)
((20, 12), 1.0361946999417073, -0.3251848939729181, 0.03005931903886999)
((40, 12), 1.0351524487072108, -0.30029592399818, 0.031945880194964874)
((20, 12), 1.0371250660111782, -0.34661006941301414, 0.030457318258924504)
((40, 12), 1.0409899857883744, -0.38583989938712615, 0.03793358494193467)
((40, 12), 1.0390952687114674, -0.3627927567342898, 0.03895320592667057)
((40, 12), 1.0365344202388107, -0.3290463075245109, 0.03101580986079454)
((20, 12), 1.0357186947386932, -0.34443193343124534, 0.02469343934617783)
((20, 12), 1.034512233762601, -0.3005883336411801, 0.02687760147804603)
caught!
((40, 12), 1.037828620572706, -0.34409706531963913, 0.033509692304340435)
caught!
((40, 12), 1.0356237750115658, -0.31701326957624426, 0.029128370016323568)
((40, 12), 1.0382771749662887, -0.35197773524019393, 0.034656560087045186)
((20, 12), 1.008674855773038, -0.0427531340477515, 0.07291939679357048)
((20, 12), 1.038329395241616, -0.36235321444030505, 0.0323028182260204)
((20, 12), 1.032701133404839, -0.28254161762681634, 0.023067449851443177)
((20, 12), 1.0341965052070046, -0.310861918737451, 0.020638836921527043)
((40, 12), 1.0329067905502771, -0.2751910183588853, 0.02703512486960155)
caught!
((40, 12), 1.0360782494144931, -0.3235455316940942, 0.029071665669892565)
caught!
((20, 12), 1.0370280483098788, -0.33824879441942385, 0.02975558202678643)
((20, 12), 1.0371640785710643, -0.34404909127877303, 0.03148683955387422)
((40, 12), 1.0388376192641926, -0.35494408907503, 0.03791242369462208)
((20, 12), 1.0331936632137442, -0.29011864245035285, 0.023526295497818005)
((40, 12), 1.036633569626667, -0.32790928354102283, 0.03329253880548233)
caught!
((40, 12), 1.0371586472788021, -0.33299902093122113, 0.03306946436011803)
((20, 12), 1.0362307208308417, -0.3249893853412822, 0.030558688477282527)
((20, 12), 1.0357191900744693, -0.31771180836941965, 0.03076657052577188)
((40, 12), 1.0384271528345543, -0.34615030429801696, 0.03794490566269941)
((40, 12), 1.0343244487316843, -0.2959128814134857, 0.028155119601275667)
((40, 12), 1.0384447907689374, -0.3607334579149925, 0.036189571920819966)
caught!
((40, 12), 1.0385806049883262, -0.3517946656353243, 0.03511272210341429)
((20, 12), 1.0395632614960713, -0.38561737361878373, 0.030723189597271737)
((20, 12), 1.0360788417443725, -0.3228045111628253, 0.030044720015704802)
((40, 12), 1.03754206117636, -0.3285586593499575, 0.03601385612375094)
((20, 12), 1.0344077431275795, -0.31030490522012666, 0.022969425176861354)
((20, 12), 1.036814333748697, -0.3393001543677613, 0.02926757637522049)
((20, 12), 1.0359502264534037, -0.329740265676541, 0.027150573256951843)
((40, 12), 1.035475837889406, -0.30473831999863826, 0.03345467951925487)
((20, 12), 1.036729596599056, -0.3370251418320701, 0.030269471629519658)
((40, 12), 1.034676894619769, -0.2973785646843761, 0.03174586392686664)
((20, 12), 1.0365059409963877, -0.3257281088934602, 0.030982047926034352)
caught!
((40, 12), 1.0363094432893267, -0.3182761356137348, 0.03428422612185954)
((20, 12), 1.0357854966659124, -0.3178421789427463, 0.030129240956421287)
((40, 12), 1.0374762390459977, -0.3470472497787744, 0.03233438237713862)
caught!
((40, 12), 1.0357849711853075, -0.3068595682482034, 0.03367979455726912)
((20, 12), 1.034810815340155, -0.303721062306297, 0.030028415138195368)
((20, 12), 1.0372858805159986, -0.3466026269144323, 0.030893285337979463)
((20, 12), 1.0371665870168258, -0.34430202627242806, 0.02967089573809186)
((40, 12), 1.0381547109875098, -0.34685752438330136, 0.036355912556502744)
((20, 12), 1.0352219012993915, -0.3093233545829345, 0.031777947093210954)
((20, 12), 1.0341354718747133, -0.29642301944736454, 0.030805849430740117)
((20, 12), 1.0378568784985585, -0.3610406218896749, 0.03045792689344451)
((20, 12), 1.0353675076719724, -0.31562040031283867, 0.03091849210944685)
((20, 12), 1.0392019988200834, -0.3752532362670539, 0.032059804639402584)
((20, 12), 1.0352703560148528, -0.3008074548472578, 0.03453065310531192)
((20, 12), 1.038078500288486, -0.3466677136812504, 0.03528617003380414)
((20, 12), 1.0359894782596006, -0.32655743929416486, 0.03040965978825864)
((20, 12), 1.0365949959950636, -0.3195082790428757, 0.034614595897471255)
((20, 12), 1.0363021349308998, -0.3335160210874537, 0.03186745785417096)
((20, 12), 1.035824434346148, -0.3184295032890183, 0.030763775824403965)
((20, 12), 1.0349111591723394, -0.302933941445727, 0.030382281653441978)
((20, 12), 1.036907781152683, -0.3392099930793868, 0.03192263858834979)
((20, 12), 1.0368189262445422, -0.33668840961245083, 0.029646065812658155)
((40, 12), 1.035603918761836, -0.30791013211327367, 0.033103938165663364)
((20, 12), 1.0328565202527258, -0.27922653289772376, 0.023903061954822016)
((20, 12), 1.0365253909922865, -0.3358989467977841, 0.030588632260751425)
caught!
((40, 12), 1.0362322151919214, -0.3271818756613229, 0.032014539244157494)
((20, 12), 1.0350578590034325, -0.3047436446901753, 0.030649554957635043)
((20, 12), 1.035973108875564, -0.3237884318169012, 0.03123553687791608)
((20, 12), 1.0343135491802338, -0.2963970108377969, 0.03005647017482315)
((20, 12), 1.039471182033987, -0.3799772903955578, 0.03247745226645989)
((20, 12), 1.0333322542338819, -0.2963339603263756, 0.02333586169779476)
((20, 12), 1.0346285921880012, -0.3145998566798684, 0.02331005702011361)
((20, 12), 1.0404386521936173, -0.41950419197351446, 0.02774962411306454)
((20, 12), 1.036716310862377, -0.3314630825509525, 0.02985816962623048)
((20, 12), 1.0358641421097974, -0.32575248102823195, 0.030206256436761884)
((20, 12), 1.0338576262272139, -0.2958794568533793, 0.023697283613117716)
((20, 12), 1.035379029878947, -0.31614627870507983, 0.030215324152147765)
((20, 12), 1.038243018883015, -0.36092903935124543, 0.03237629810870338)
((20, 12), 1.0330140066412334, -0.29261421998745935, 0.023911517774180564)
((20, 12), 1.0370968456207637, -0.3468591236655321, 0.02997773628136986)
caught!
((20, 12), 1.033158326015655, -0.2814316025296618, 0.02381723342131621)
((20, 12), 1.035234658382589, -0.31290818949046734, 0.030432818172086734)
((20, 12), 1.0334080820470153, -0.29426747963214017, 0.023425147563241634)
((20, 12), 1.0350995667787894, -0.3078837837713716, 0.031534003651595775)
((20, 12), 1.0339241957823802, -0.3025679753587935, 0.023639220986414376)
((20, 12), 1.0370282496626, -0.34180345417508223, 0.031060961661410154)
((20, 12), 1.0355768956461366, -0.3140125898434718, 0.030084022094779273)
((20, 12), 1.0378566529938007, -0.3499900557148449, 0.03076406604012131)
((20, 12), 1.0358389681442606, -0.3172205962457506, 0.030463652180526102)
((20, 12), 1.0371337262854885, -0.3453425958323992, 0.030627092558647217)
((20, 12), 1.0367898160328055, -0.3333834358553511, 0.031116487505876346)
caught!
((20, 12), 1.0361555433806173, -0.341615870027734, 0.02304457681006055)
((20, 12), 1.033057306041249, -0.294395312987969, 0.022996049841825055)
((20, 12), 1.0354661764113733, -0.31988554179766626, 0.02998309157145045)
((20, 12), 1.0358115099198832, -0.3198300775062399, 0.030015874447571388)
((20, 12), 1.0343482348088682, -0.30112207078727976, 0.023961029097509474)
((20, 12), 1.035375982866453, -0.3121698611252701, 0.03170979092690099)
((20, 12), 1.0334227603783197, -0.295639320456149, 0.02335003984053668)
((20, 12), 1.032617100315174, -0.27792376015576953, 0.023237653796772503)
((20, 12), 1.0314988833669463, -0.2472008942510758, 0.020958660386311615)
((20, 12), 1.039622416188808, -0.3866603630040733, 0.029987969017888267)
((20, 12), 1.0328209461449396, -0.28131456969028246, 0.020790314925631884)
((20, 12), 1.0354790988243927, -0.313906780300502, 0.03072323232244875)
((20, 12), 1.0330411818076606, -0.28794047341845663, 0.02266664560894313)
((20, 12), 1.0339424420639456, -0.30799597560107866, 0.02303978627467549)
((20, 12), 1.0325182431752964, -0.2759952111171059, 0.023201910537498753)
((20, 12), 1.0351860716723975, -0.304352136197505, 0.031080944511972852)
((20, 12), 1.0323614806663168, -0.26950985427124896, 0.023733452179813402)
((20, 12), 1.0344312061408698, -0.3011130815505141, 0.031038693006083064)
((20, 12), 1.0346045829235677, -0.3155557883412414, 0.022881285173980227)
((20, 12), 1.036995834394781, -0.341850191848193, 0.030670240237988422)
((20, 12), 1.03221382514693, -0.27654270338811116, 0.02063672520924998)
((20, 12), 1.0346609512969593, -0.3135119816244941, 0.02322757719529834)
((20, 12), 1.033245108960176, -0.2933871215042093, 0.022869347205904562)
((20, 12), 1.0323264696006649, -0.27721811928893425, 0.02311241210313504)
((20, 12), 1.033963343522022, -0.2842899518380864, 0.029557600009629065)
((20, 12), 1.0383389585120337, -0.3467004193963629, 0.034864419965726735)
((20, 12), 1.038213947458497, -0.3517439730338931, 0.03453233303049026)
((20, 12), 1.0360218924773392, -0.3155835830888482, 0.03184919404041852)

In [21]:

    

# myArr[myArr > myNumber].min()
# myArr[myArr < myNumber].max()
def nearest_ind(items, pivot):
    indices = []
    earlier_indices = items[items>pivot] #find earlier reference
    later_indices = items[items<pivot]  #find later reference
    if len(earlier_indices)>0:
        indices.append(earlier_indices.idxmin())
    if len(later_indices)>0:
        indices.append(later_indices.idxmax())
    return indices


df = get_lcmsruns()
df['coefficient'] = pd.to_numeric(df['coefficient'])
no_coef = df[pd.isnull(df['coefficient'])]
coef = df[~pd.isnull(df['coefficient'])]
coef.reset_index(inplace=True,drop=True)
no_coef.reset_index(inplace=True,drop=True)
for i,row in no_coef.head().iterrows():
    idx = nearest_ind(coef.loc[coef['experiment']==row['experiment'],'date_finished'],row['date_finished'])
#     print(row['name'],len(idx))
#     print(i,len(idx),row['date_finished'],np.mean([float(coef.loc[i,'intercept']) for i in idx]),)
    new_data = {'name':row['name'],
                'coefficient':np.mean([float(coef.loc[i,'coefficient']) for i in idx]),
                'intercept': np.mean([float(coef.loc[i,'intercept']) for i in idx]),
                'reference_rt_space':'QCsopV3'}
    print(idx,new_data)
#     insertResult = lk.query.update_rows(pt_con, 'samples', 'lcmsrun_sample', [new_data])


    

    




execute_sql: There are 1140 rows.
([0], {'coefficient': 0.9926166534423828, 'intercept': -0.1267438530921936, 'name': u'20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HILIC_USHGX01163_FPS_MS1_0_InjBL-MeOH_Post_Rg70to1050---InjBL-MeOH_Run423.mzML', 'reference_rt_space': 'QCsopV3'})
([0], {'coefficient': 0.9926166534423828, 'intercept': -0.1267438530921936, 'name': u'20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HILIC_USHGX01163_FPS_MS1_0_InjBL-MeOH_Post_Rg70to1050---InjBL-MeOH_Run424.mzML', 'reference_rt_space': 'QCsopV3'})
([0], {'coefficient': 0.9926166534423828, 'intercept': -0.1267438530921936, 'name': u'20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HILIC_USHGX01163_FPS_MS1_0_InjBL-MeOH_Post_Rg70to1050---InjBL-MeOH_Run425.mzML', 'reference_rt_space': 'QCsopV3'})
([1], {'coefficient': 0.9929835200309752, 'intercept': -0.10319431871175766, 'name': u'20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HILIC_USHGX01163_FPS_MS1_0_InjBL-MeOH_Pre_Rg70to1050---InjBL-MeOH_Run1.mzML', 'reference_rt_space': 'QCsopV3'})
([3, 1], {'coefficient': 0.9934036135673523, 'intercept': -0.10953303426504135, 'name': u'20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HILIC_USHGX01163_FPS_MS1_0_InjBL-MeOH_Pre_Rg70to1050---InjBL-MeOH_Run10.mzML', 'reference_rt_space': 'QCsopV3'})

In [113]:

    

df = get_lcmsruns()
# idx = (df['name'].str.contains('_JP_')) & (df['name'].str.contains('_QC_'))
# print(sum(idx))
# plt.close('all')
# fig,ax =plt.subplots()
# x = np.asarray(df[idx].date_finished)
# y = np.asarray(df[idx].intercept.apply(lambda x: float(x)))
# c = np.asarray(df[idx].mean_squared_error.apply(lambda x: float(x)))
# avg = np.sum(y/c)/np.sum(1/c)
# ax.plot(x,y,'.')
# ax.axhline(avg)
# plt.show()


    

    




execute_sql: There are 1140 rows.

In [118]:

    

df[idx]


    

    
Out[118]:







  

    

      

      
RowId
      
_labkeyurl_RowId
      
_labkeyurl_mass_spectrometry
      
_labkeyurl_name
      
coefficient
      
date_finished
      
hdf5_file
      
intercept
      
mass_spectrometry
      
mean_squared_error
      
name
      
experiment
    
  
  

    

      
0
      
31956
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-10 14:40:45
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
1
      
31957
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-10 15:02:48
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
2
      
31958
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-10 15:24:52
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
3
      
31959
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-04 02:11:43
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
4
      
31960
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-04 05:30:16
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
5
      
31961
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-04 02:33:49
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
6
      
31962
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-04 02:55:51
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
7
      
31963
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-04 03:17:54
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
8
      
31964
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-04 03:40:01
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
9
      
31965
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-04 04:46:10
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
10
      
31966
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-04 05:08:13
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
11
      
31967
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03883767127990723
      
2019-01-10 14:18:42
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.354944080114364624
      
MS1-v1_FPS
      
0.037912
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
12
      
31968
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03350841999053955
      
2019-01-04 04:24:07
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.281171500682830811
      
MS1-v1_FPS
      
0.026494
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
13
      
31969
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
0.992616653442382812
      
2019-01-10 13:56:39
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.126743853092193604
      
MS1-v1_FPS
      
0.045596
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
14
      
31970
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
0.992983520030975342
      
2019-01-04 04:02:06
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.10319431871175766
      
MS1-v1_FPS
      
0.043135
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
15
      
31971
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-10 10:16:16
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
16
      
31972
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-10 11:22:22
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
17
      
31973
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-10 12:28:29
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
18
      
31974
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-10 13:34:38
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
19
      
31975
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-04 06:36:23
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
20
      
31976
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-06 11:53:21
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
21
      
31977
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-08 00:16:39
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
22
      
31978
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-09 09:21:38
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
23
      
31979
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-04 07:42:32
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
24
      
31980
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03560662269592285
      
2019-01-07 00:45:04
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.308855801820755005
      
MS1-v1_FPS
      
0.031930
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
25
      
31981
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-08 10:56:10
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
26
      
31982
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03290677070617676
      
2019-01-04 08:48:41
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.275191009044647217
      
MS1-v1_FPS
      
0.027035
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
27
      
31983
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-08 18:17:14
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
28
      
31984
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-07 12:30:41
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
29
      
31985
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
None
      
2019-01-04 09:54:46
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
None
      
MS1-v1_FPS
      
NaN
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
...
      
...
      
...
      
...
      
...
      
...
      
...
      
...
      
...
      
...
      
...
      
...
      
...
    
    

      
395
      
32351
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03433811664581299
      
2019-01-05 14:56:35
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.300570636987686157
      
MSMS-v2_POS
      
0.028126
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
396
      
32352
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03421413898468018
      
2019-01-06 13:21:36
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.2984352707862854
      
MSMS-v2_POS
      
0.027566
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
397
      
32353
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03807854652404785
      
2019-01-09 11:56:11
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.346667706966400146
      
MSMS-v2_POS
      
0.035286
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
398
      
32354
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03535056114196777
      
2019-01-05 16:02:41
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.307731479406356812
      
MSMS-v2_POS
      
0.028890
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
399
      
32355
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03375875949859619
      
2019-01-06 02:41:58
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.300200104713439941
      
MSMS-v2_POS
      
0.027112
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
400
      
32356
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03355956077575684
      
2019-01-09 07:31:22
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.282362848520278931
      
MSMS-v2_POS
      
0.027206
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
401
      
32357
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03438091278076172
      
2019-01-05 17:08:45
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.304301917552947998
      
MSMS-v2_POS
      
0.027481
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
402
      
32358
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03481447696685791
      
2019-01-06 12:15:24
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.309899717569351196
      
MSMS-v2_POS
      
0.028159
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
403
      
32359
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03360569477081299
      
2019-01-05 18:14:55
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.288130521774291992
      
MSMS-v2_POS
      
0.027146
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
404
      
32360
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03582203388214111
      
2019-01-07 13:58:56
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.324347466230392456
      
MSMS-v2_POS
      
0.027178
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
405
      
32361
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03618991374969482
      
2019-01-05 19:21:01
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.321991264820098877
      
MSMS-v2_POS
      
0.028518
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
406
      
32362
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.0339350700378418
      
2019-01-06 14:27:43
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.298027217388153076
      
MSMS-v2_POS
      
0.027140
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
407
      
32363
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03552043437957764
      
2019-01-08 12:24:24
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.319721907377243042
      
MSMS-v2_POS
      
0.027534
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
408
      
32364
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03397870063781738
      
2019-01-05 20:27:10
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.300447642803192139
      
MSMS-v2_POS
      
0.027171
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
409
      
32365
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03418123722076416
      
2019-01-06 18:52:10
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.304282009601593018
      
MSMS-v2_POS
      
0.026997
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
410
      
32366
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03506112098693848
      
2019-01-08 01:44:52
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.318362116813659668
      
MSMS-v2_POS
      
0.026941
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
411
      
32367
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03645789623260498
      
2019-01-08 21:57:42
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.334086388349533081
      
MSMS-v2_POS
      
0.027221
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
412
      
32368
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03938472270965576
      
2019-01-09 22:52:45
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.375489383935928345
      
MSMS-v2_POS
      
0.035554
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
413
      
32369
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.04022955894470215
      
2019-01-10 01:05:07
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.385877162218093872
      
MSMS-v2_POS
      
0.037108
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
414
      
32370
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03521668910980225
      
2019-01-05 21:33:15
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.314087778329849243
      
MSMS-v2_POS
      
0.027261
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
415
      
32371
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.0403130054473877
      
2019-01-09 20:40:29
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.389493346214294434
      
MSMS-v2_POS
      
0.035361
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
416
      
32372
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03433871269226074
      
2019-01-07 10:40:26
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.301980704069137573
      
MSMS-v2_POS
      
0.027578
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
417
      
32373
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03778910636901855
      
2019-01-10 07:19:54
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.360440284013748169
      
MSMS-v2_POS
      
0.033910
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
418
      
32374
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03676736354827881
      
2019-01-09 00:10:01
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.346258074045181274
      
MSMS-v2_POS
      
0.026823
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
419
      
32375
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03454124927520752
      
2019-01-05 22:39:27
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.309071600437164307
      
MSMS-v2_POS
      
0.027037
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
420
      
32376
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.0340801477432251
      
2019-01-07 03:19:26
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.302315503358840942
      
MSMS-v2_POS
      
0.027328
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
421
      
32377
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03554391860961914
      
2019-01-09 13:02:19
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.328685343265533447
      
MSMS-v2_POS
      
0.027619
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
422
      
32378
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03608715534210205
      
2019-01-08 02:51:01
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.330770552158355713
      
MSMS-v2_POS
      
0.027079
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
423
      
32379
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03925120830535889
      
2019-01-10 06:13:45
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.376722216606140137
      
MSMS-v2_POS
      
0.035134
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
    

      
424
      
32380
      
/Project%20Tracking/experiment-showMaterial.vi...
      
/Project%20Tracking/list-details.view?listId=6...
      
/Project%20Tracking/experiment-showMaterial.vi...
      
1.03669130802154541
      
2019-01-07 22:26:21
      
/project/projectdirs/metatlas/raw_data/akuftin...
      
-0.343997716903686523
      
MSMS-v2_POS
      
0.027112
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
      
20190103_KBL_JP_502924_Soil_13CFlux_QE-139_HIL...
    
  

425 rows × 12 columns

In [138]:

    

df = get_lcmsruns()
idx = (df['name'].str.contains('_JP_')) & (pd.to_numeric(df['coefficient'],'force')>0) &  (~df['name'].str.contains('-QC_'))
print(sum(idx))
plt.close('all')
fig,ax =plt.subplots()
x = np.asarray(df[idx].date_finished)
y = np.asarray(df[idx].intercept.apply(lambda x: float(x)))
c = np.asarray(df[idx].mean_squared_error.apply(lambda x: float(x)))
avg = np.sum(y/c)/np.sum(1/c)
ax.plot(x,y,'.')
plt.axhline(avg,color='red')
ax.set_ylim(-0.45,0)
plt.show()


    

    




execute_sql: There are 1140 rows.
299






    




Traceback (most recent call last):
  File "/global/common/software/m2650/python-cori/lib/python2.7/site-packages/matplotlib/cbook/__init__.py", line 388, in process
    proxy(*args, **kwargs)
  File "/global/common/software/m2650/python-cori/lib/python2.7/site-packages/matplotlib/cbook/__init__.py", line 228, in __call__
    return mtd(*args, **kwargs)
  File "/global/common/software/m2650/python-cori/lib/python2.7/site-packages/matplotlib/backends/backend_nbagg.py", line 241, in <lambda>
    canvas.mpl_connect('close_event', lambda event: Gcf.destroy(num))
  File "/global/common/software/m2650/python-cori/lib/python2.7/site-packages/matplotlib/_pylab_helpers.py", line 58, in destroy
    cls._activeQue.remove(manager)
ValueError: list.remove(x): x not in list






    















    

In [141]:

    

df = get_lcmsruns()
idx = (df['name'].str.contains('_KZ_')) & (pd.to_numeric(df['coefficient'],'force')>0) &  (~df['name'].str.contains('_QC'))
print(sum(idx))
plt.close('all')
fig,ax =plt.subplots()
x = np.asarray(df[idx].date_finished)
y = np.asarray(df[idx].intercept.apply(lambda x: float(x)))
c = np.asarray(df[idx].mean_squared_error.apply(lambda x: float(x)))
avg = np.sum(y/c)/np.sum(1/c)
ax.plot(x,y,'.')
ax.axhline(avg,color='red')
plt.show()


    

    




execute_sql: There are 1140 rows.
31






    















    

In [106]:

    

df = get_lcmsruns()
idx = (df['name'].str.contains('_KZ_')) & (df['name'].str.contains('_QC'))
print(sum(idx))
# fig,ax =plt.subplots()
x = np.asarray(df[idx].date_finished)
y = np.asarray(df[idx].intercept.apply(lambda x: float(x)))
c = np.asarray(df[idx].mean_squared_error.apply(lambda x: float(x)))
avg = np.sum(y/c)/np.sum(1/c)

plt.close('all')
plt.scatter(x,y,c=c)
plt.axhline(avg)
plt.show()


    

    




execute_sql: There are 1140 rows.
31






    




Traceback (most recent call last):
  File "/global/common/software/m2650/python-cori/lib/python2.7/site-packages/matplotlib/cbook/__init__.py", line 388, in process
    proxy(*args, **kwargs)
  File "/global/common/software/m2650/python-cori/lib/python2.7/site-packages/matplotlib/cbook/__init__.py", line 228, in __call__
    return mtd(*args, **kwargs)
  File "/global/common/software/m2650/python-cori/lib/python2.7/site-packages/matplotlib/backends/backend_nbagg.py", line 241, in <lambda>
    canvas.mpl_connect('close_event', lambda event: Gcf.destroy(num))
  File "/global/common/software/m2650/python-cori/lib/python2.7/site-packages/matplotlib/_pylab_helpers.py", line 58, in destroy
    cls._activeQue.remove(manager)
ValueError: list.remove(x): x not in list






    















    

In [ ]:

    

idx = df['name'].str.contains('_KZ_')
plt.plot(df[idx].date_finished,df[idx].intercept.apply(lambda x: float(x)),'.')


    

In [ ]:

    




    

In [ ]:

    

df = get_reference_mzrt()
df.head()


    

In [ ]:

    

# df = get_lcmsruns(filename='%POS%')
df = get_lcmsruns()

df.head()


    

In [ ]:

    

# For each QC file, get the slope and intercept


    

In [ ]:

    




    

In [22]:

    

plt.ioff()
colors = ['red','black']
for i,f in enumerate(df.hdf5_file.unique()):
    fig,ax = plt.subplots(nrows=1,ncols=1,figsize=(12,12))

    small_df = df[df.hdf5_file==f]
    small_df = small_df[small_df['mz_centroid']!='NaN']
    small_df = small_df[small_df['peak_height']>1e6]
    if len(small_df.polarity.unique())>1:
        for j,p in enumerate(['positive','negative']):
            
            x = small_df[small_df['polarity']==p].mz.values
            y = small_df[small_df['polarity']==p].mz_centroid.values
            temp = np.divide(x-y,x)*1e6
            bins = np.linspace(-20,20,41)
            ax.hist(np.reshape(temp,(len(temp),1)),bins=bins,color=colors[j],alpha=0.6)
            ax.set_title(os.path.basename(f))
    else:
        x = small_df[small_df['polarity']==p].mz.values
        y = small_df[small_df['polarity']==p].mz_centroid.values
        temp = np.divide(x-y,x)*1e6
        bins = np.linspace(-20,20,41)
        ax.hist(np.reshape(temp,(len(temp),1)),bins=bins,color='blue',alpha=1)
    fig.savefig('/global/homes/b/bpb/Downloads/qcfigs/%s.pdf'%(os.path.basename(f).split('.')[0]))
    plt.close('all')
plt.ion()


    

    





KeyErrorTraceback (most recent call last)
<ipython-input-22-a90dc757b562> in <module>()
      5 
      6     small_df = df[df.hdf5_file==f]
----> 7     small_df = small_df[small_df['mz_centroid']!='NaN']
      8     small_df = small_df[small_df['peak_height']>1e6]
      9     if len(small_df.polarity.unique())>1:

/global/common/software/m2650/python-cori/lib/python2.7/site-packages/pandas/core/frame.pyc in __getitem__(self, key)
   2137             return self._getitem_multilevel(key)
   2138         else:
-> 2139             return self._getitem_column(key)
   2140 
   2141     def _getitem_column(self, key):

/global/common/software/m2650/python-cori/lib/python2.7/site-packages/pandas/core/frame.pyc in _getitem_column(self, key)
   2144         # get column
   2145         if self.columns.is_unique:
-> 2146             return self._get_item_cache(key)
   2147 
   2148         # duplicate columns & possible reduce dimensionality

/global/common/software/m2650/python-cori/lib/python2.7/site-packages/pandas/core/generic.pyc in _get_item_cache(self, item)
   1840         res = cache.get(item)
   1841         if res is None:
-> 1842             values = self._data.get(item)
   1843             res = self._box_item_values(item, values)
   1844             cache[item] = res

/global/common/software/m2650/python-cori/lib/python2.7/site-packages/pandas/core/internals.pyc in get(self, item, fastpath)
   3841 
   3842             if not isna(item):
-> 3843                 loc = self.items.get_loc(item)
   3844             else:
   3845                 indexer = np.arange(len(self.items))[isna(self.items)]

/global/common/software/m2650/python-cori/lib/python2.7/site-packages/pandas/core/indexes/base.pyc in get_loc(self, key, method, tolerance)
   2525                 return self._engine.get_loc(key)
   2526             except KeyError:
-> 2527                 return self._engine.get_loc(self._maybe_cast_indexer(key))
   2528 
   2529         indexer = self.get_indexer([key], method=method, tolerance=tolerance)

pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()

pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()

pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item()

pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: 'mz_centroid'

In [23]:

    

atlas = processing_inputs['QCsopV3']['atlas_pos']
filenames = processing_inputs['QCsopV3']['files_pos']
input_data = setup_file_slicing(atlas,filenames)

ft = reload(ft)
d = ft.get_data(input_data[1],return_data=True,save_file=False)

ms1_summary = d['ms1_summary']
measured_signals = ms1_summary[ms1_summary['peak_height']>1e6].sort_values('label')
df = pd.merge(measured_signals,processing_inputs['QCsopV3']['atlas_pos'][['label','rt_peak']],left_on='label',right_on='label',how='left',suffixes=['_measured','_theoretical'])

x = df['rt_peak_theoretical'].values.reshape(-1, 1)
y = df['rt_peak_measured'].values

# Fit line using all data
lr = linear_model.LinearRegression()
lr.fit(x, y)

# Robustly fit linear model with RANSAC algorithm
ransac = linear_model.RANSACRegressor()
ransac.fit(x, y)
inlier_mask = ransac.inlier_mask_
outlier_mask = np.logical_not(inlier_mask)

# Predict data of estimated models
line_X = np.arange(x.min(), x.max())[:, np.newaxis]
line_y = lr.predict(line_X)
line_y_ransac = ransac.predict(line_X)

# fit using a Huber Regressor
epsilon_values = [1.35, 1.5, 1.75, 1.9]
for k, epsilon in enumerate(epsilon_values):
    huber = HuberRegressor(fit_intercept=True, alpha=0.0, max_iter=100,
                       epsilon=epsilon)
    huber.fit(x, y)
    print('Huber',huber.coef_,huber.intercept_)

# Compare estimated coefficients
print("Estimated coefficients (true, linear regression, RANSAC):")
print(lr.coef_, ransac.estimator_.coef_)
print(lr.intercept_,ransac.estimator_.intercept_,)

lw = 2
plt.scatter(x[inlier_mask], y[inlier_mask], color='yellowgreen', marker='.',
            label='Inliers')
plt.scatter(x[outlier_mask], y[outlier_mask], color='gold', marker='.',
            label='Outliers')
plt.plot(line_X, line_y, color='navy', linewidth=lw, label='Linear regressor')
plt.plot(line_X, line_y_ransac, color='cornflowerblue', linewidth=lw,
         label='RANSAC regressor')
plt.legend(loc='lower right')
plt.xlabel("Theoretical")
plt.ylabel("Measured")
plt.show()


    

    





KeyErrorTraceback (most recent call last)
<ipython-input-23-d3d7ecb1671a> in <module>()
----> 1 atlas = processing_inputs['QCsopV3']['atlas_pos']
      2 filenames = processing_inputs['QCsopV3']['files_pos']
      3 input_data = setup_file_slicing(atlas,filenames)
      4 
      5 ft = reload(ft)

KeyError: 'QCsopV3'

In [ ]:

    

# filters = [
#     lk.query.QueryFilter('inchi_key', 'RWQNBRDOKXIBIV-UHFFFAOYSA-N'),
#     lk.query.QueryFilter('precursor_mz', '125, 125.5', lk.query.QueryFilter.Types.BETWEEN),
#     lk.query.QueryFilter('database','mo',lk.query.QueryFilter.Types.CONTAINS)
#     ]

# schema = 'lists'
# table = 'ReferenceMSMS'
# result = lk.query.select_rows(con, schema, table, filter_array=filters)

# if result is not None:
#     print("select_rows: There are " + str(result['rowCount']) + " rows.")
#     df = pd.DataFrame(result['rows'])
# else:
#     print('select_rows: Failed to load results from ' + schema + '.' + table)
# df


    

In [ ]:

    

###################
# Test execute_sql
###################
sql = """SELECT 
msms.collision_energy,
  cpd.name
FROM ReferenceMSMS AS msms
INNER JOIN MetAtlasCompounds AS cpd ON msms.inchi_key  = cpd.inchi_key
WHERE msms.inchi_key ='RWQNBRDOKXIBIV-UHFFFAOYSA-N';"""

# sql = """SELECT msms.collision_energy FROM ReferenceMSMS AS msms WHERE msms.inchi_key ='CQOVPNPJLQNMDC-ZETCQYMHSA-N';"""

# sql = """SELECT * FROM ReferenceMSMS WHERE inchi_key ='CQOVPNPJLQNMDC-ZETCQYMHSA-N';"""
# sql = """SELECT * FROM Compounds WHERE inchikey LIKE 'CQOVPNPJLQNMDC%';"""

labkey_server = 'metabolomics.app.labkey.host'
project_name = 'Metabolite Atlas'  # Project folder name
schema = 'lists'
con = lk.utils.create_server_context(labkey_server, project_name, use_ssl=True)


# base execute_sql
sql_result = lk.query.execute_sql(con, schema, sql)
if sql_result is not None:
    print("execute_sql: There are " + str(sql_result['rowCount']) + " rows.")
else:
    print('execute_sql: Failed to load results from ' + schema + '.' + table)
pd.DataFrame(sql_result['rows'])
# # paging
# sql_result = execute_sql(server_context, schema, sql, max_rows=5, offset=10,
#                          sort=(column1 + ', -' + column2))
# if sql_result is not None:
#     print('execute_sql: There are ' + str(len(sql_result['rows'])) + ' rows.')
#     print('execute_sql: There are ' + str(sql_result['rowCount']) + ' total rows.')
#     print('execute_sql: ' + table)
#     for row in sql_result['rows']:
#         print('\t' + str(row[column1]) + ', ' + str(row[column2]))
# else:
#     print('execute_sql: Failed to load results from ' + schema + '.' + table)

# # Save query within the session
# sql_result = execute_sql(server_context, schema, sql, max_rows=5, offset=10,
#                          save_in_session=True)
# print('execute_sql: query saved as [ ' + sql_result['queryName'] + ' ]')


    

In [ ]:

    

###################
# Test execute_sql
###################
sql = """SELECT 
msms.collision_energy,
  cpd.name
FROM ReferenceMSMS AS msms
INNER JOIN MetAtlasCompounds AS cpd ON msms.inchi_key  = cpd.inchi_key
WHERE msms.inchi_key ='RWQNBRDOKXIBIV-UHFFFAOYSA-N';"""

# sql = """SELECT msms.collision_energy FROM ReferenceMSMS AS msms WHERE msms.inchi_key ='CQOVPNPJLQNMDC-ZETCQYMHSA-N';"""

# sql = """SELECT * FROM ReferenceMSMS WHERE inchi_key ='CQOVPNPJLQNMDC-ZETCQYMHSA-N';"""
# sql = """SELECT * FROM Compounds WHERE inchikey LIKE 'CQOVPNPJLQNMDC%';"""

labkey_server = 'metabolomics.app.labkey.host'
project_name = 'Metabolite Atlas'  # Project folder name
schema = 'lists'
con = lk.utils.create_server_context(labkey_server, project_name, use_ssl=True)#,api_key=key)


# base execute_sql
sql_result = lk.query.execute_sql(con, schema, sql)
if sql_result is not None:
    print("execute_sql: There are " + str(sql_result['rowCount']) + " rows.")
else:
    print('execute_sql: Failed to load results from ' + schema + '.' + table)
pd.DataFrame(sql_result['rows'])
# # paging
# sql_result = execute_sql(server_context, schema, sql, max_rows=5, offset=10,
#                          sort=(column1 + ', -' + column2))
# if sql_result is not None:
#     print('execute_sql: There are ' + str(len(sql_result['rows'])) + ' rows.')
#     print('execute_sql: There are ' + str(sql_result['rowCount']) + ' total rows.')
#     print('execute_sql: ' + table)
#     for row in sql_result['rows']:
#         print('\t' + str(row[column1]) + ', ' + str(row[column2]))
# else:
#     print('execute_sql: Failed to load results from ' + schema + '.' + table)

# # Save query within the session
# sql_result = execute_sql(server_context, schema, sql, max_rows=5, offset=10,
#                          save_in_session=True)
# print('execute_sql: query saved as [ ' + sql_result['queryName'] + ' ]')


    

In [ ]: