This notebook was used for developing a script to translate MagIC format files from the 2.5 data format to the 3.0 format. This functionality is now implemented in Pmag GUI.
In [1]:
import pmagpy.new_builder as nb
from pmagpy import ipmag
import os
import json
import numpy as np
import sys
import pandas as pd
import numpy as np
from pandas import DataFrame
from pmagpy import builder2 as builder
from pmagpy import validate_upload2 as validate_upload
from pmagpy import pmag
from pmagpy.mapping import map_magic
from pmagpy import pmag
reload(nb)
reload(map_magic)
WD = os.path.realpath(os.path.join("..", "2_5", "McMurdo"))
In [2]:
# convert magic_measurements to measurements (3.0)
# first unpack lawrence et al., 2009 datafile from MagIC
!download_magic.py -f zmab0100049tmp03.txt -WD ../2_5/McMurdo -ID ../2_5/McMurdo
In [3]:
# read in data model 2.5 measruements file
data2,filetype = pmag.magic_read(WD+'/magic_measurements.txt')
print filetype, len(data2)
In [4]:
NewMeas = []
# step through records
for rec in data2:
NewMeas.append(map_magic.convert_meas('magic3',rec))
pmag.magic_write(WD+'/measurements.txt',NewMeas,'measurements')
Out[4]:
In [5]:
reload(nb)
reload(pmag)
Out[5]:
In [6]:
WD = os.path.join("..", "2_5", "McMurdo")
#for dtype in ['specimens', 'samples', 'sites', 'locations']:
# filename = os.path.join(WD, '{}.txt'.format(dtype))
# if os.path.exists(filename):
# os.remove(filename)
In [7]:
# convert magic_measurements file only
new_meas, upgraded, no_upgrade = pmag.convert_directory_2_to_3("magic_measurements.txt", WD, WD, meas_only=True)
In [8]:
# create a contribution using the converted measurement data
con = nb.Contribution(WD, read_tables=['measurements'])
# use name data in measurement table to create specimen-location tables
con.propagate_measurement_info()
# show sample table created from measurement info
con.tables['samples'].df.head()
Out[8]:
In [9]:
# convert a pandas DataFrame to the standard PmagPy formats:
# either a dict of dicts or a list of dicts, each corresponding to one table row
def convert_to_pmag_data_list(df, lst_or_dict):
"""
"""
dictionary = dict(df.T)
if lst_or_dict == "lst":
return [dict(dictionary[key]) for key in dictionary]
else:
return {key: dict(dictionary[key]) for key in dictionary}
site_df = con.tables['sites'].df.head()
print convert_to_pmag_data_list(site_df, "dict")
print convert_to_pmag_data_list(site_df, "lst")
In [10]:
import pmagpy.mapping.map_magic as mm
import pmagpy.new_builder as nb
reload(mm)
reload(nb)
reload(pmag)
wdir = os.path.join("..", "2_5", "McMurdo")
# take er_*.txt files and pmag_*.txt files, combine them, then turn them to 3.0. and write them out
dtype = "specimens"
map_dict = mm.spec_magic2_2_magic3_map
pmag.convert_and_combine_2_to_3(dtype, map_dict, input_dir=wdir, output_dir=wdir)
nb.MagicDataFrame(os.path.join(wdir, "{}.txt".format(dtype))).df.head()
Out[10]:
In [11]:
# converts measurements file and any present specimen, sample, site, or location files to 3.0.
# does not yet handle any other MagIC format files
new_meas, upgraded, not_upgraded = pmag.convert_directory_2_to_3('magic_measurements.txt', wdir, wdir)
print 'upgraded files: {}'.format(', '.join(upgraded))
print 'files that could not be upgraded: {}'.format(', '.join(not_upgraded))
In [12]:
import pmagpy.new_builder as nb
import pmagpy.data_model3 as data_model
con = nb.Contribution('../3_0/Megiddo', dmodel=data_model.DataModel())
In [13]:
site_dm = con.data_model.dm['sites']
site_dm['name'] = site_dm.index
site_dm[['name', 'type']].head()
Out[13]:
In [14]:
dtypes = set()
for dm_name in con.data_model.dm:
dtypes = dtypes.union(con.data_model.dm[dm_name]['type'].unique())
print ", ".join(dtypes)
In [15]:
site_df = con.tables['sites'].df
for col_name in site_df.columns:
dtype = site_dm.loc[col_name, 'type']
if dtype == 'Number':
site_df[col_name] = site_df[col_name].astype(float)
elif dtype == 'Integer':
site_df[col_name] = site_df[col_name].fillna(0)
site_df[col_name] = site_df[col_name].astype(int)
#site_df[col_name] = site_df[col_name].replace(-999, np.nan) # can't have dtype of int & np.nan/None values
elif dtype == 'String':
#print "string", col_name
site_df[col_name] = site_df[col_name].astype(str) # can't have dtype of str & np.nan/None values
#site_df[col_name] == site_df[col_name].astype(int)
for col in ['age', 'dir_n_samples', 'criteria']:
print col, ":", site_df[col].dtype
In [16]:
reload(pmag)
#pmag.convert_measfile_2_to_3('magic_measurements.txt', '2_5/McMurdo')
fname = os.path.join("..", '3_0', 'Megiddo', 'sites.txt')
df = nb.MagicDataFrame(os.path.join("..", '3_0', 'Megiddo', 'sites.txt')).df
pmag.magic_read(fname)
df = pd.read_table(fname, skiprows=[0])
df['age'].astype(str).head()
Out[16]: