In [24]:
import pandas as pd

In [25]:
# source: https://n5eil01u.ecs.nsidc.org/DP4/ICEBRIDGE/IGGRV1B.001/2014.11.22/IGGRV1B_20141122_11422600_V016.txt
# date: 2014-11-22T11:42:26Z to 2014-11-23T00:19:05Z
df_1 = pd.read_csv("../data/IGGRV1B_20141122_11422600_V016.txt", comment='#', delim_whitespace=True)

In [26]:
df_1.head()


Out[26]:
LAT LONG DATE DOY TIME FLT PSX PSY WGSHGT FX FY FZ EOTGRAV FACOR INTCOR FAG070 FAG100 FAG140 FLTENVIRO
0 -53.004553 -70.845650 20141122 326 42146.0 522 -3929448.42 1364869.24 50.15 2.03 2.91 981296.66 NaN NaN NaN NaN NaN NaN -1
1 -53.004553 -70.845650 20141122 326 42146.5 522 -3929448.40 1364869.24 50.16 2.24 2.29 981296.95 NaN NaN NaN NaN NaN NaN -1
2 -53.004554 -70.845650 20141122 326 42147.0 522 -3929448.38 1364869.22 50.16 3.00 1.93 981297.26 NaN NaN NaN NaN NaN NaN -1
3 -53.004554 -70.845651 20141122 326 42147.5 522 -3929448.36 1364869.21 50.16 3.93 2.01 981297.39 NaN NaN NaN NaN NaN NaN -1
4 -53.004554 -70.845651 20141122 326 42148.0 522 -3929448.33 1364869.20 50.16 4.61 2.44 981297.24 NaN NaN NaN NaN NaN NaN -1

In [27]:
df_confirmed_data = df_1[df_1['FLTENVIRO'] == 0]
df_confirmed_data.head()


Out[27]:
LAT LONG DATE DOY TIME FLT PSX PSY WGSHGT FX FY FZ EOTGRAV FACOR INTCOR FAG070 FAG100 FAG140 FLTENVIRO
9219 -53.113036 -70.933760 20141122 326 46755.5 522 -3919206.16 1354561.13 1759.89 -18632.95 -40848.38 908527.64 -609.23 542.68 0.0 -11.74 -11.20 -11.14 0
9220 -53.113503 -70.933209 20141122 326 46756.0 522 -3919140.06 1354580.48 1762.76 -18721.43 -47234.06 882070.87 -765.59 543.57 0.0 -11.92 -11.34 -11.27 0
9221 -53.113971 -70.932657 20141122 326 46756.5 522 -3919073.85 1354599.85 1765.39 -18936.33 -53345.99 863665.66 -845.74 544.38 0.0 -12.09 -11.48 -11.41 0
9222 -53.114439 -70.932104 20141122 326 46757.0 522 -3919007.48 1354619.26 1767.73 -19714.64 -58528.23 854739.16 -845.87 545.10 0.0 -12.26 -11.62 -11.54 0
9223 -53.114909 -70.931550 20141122 326 46757.5 522 -3918940.94 1354638.69 1769.75 -21445.33 -62462.80 854421.13 -796.82 545.72 0.0 -12.42 -11.76 -11.68 0

In [28]:
df_confirmed_data = df_confirmed_data[['LAT', 'LONG', 'DATE', 'FAG100']]
df_confirmed_data.head()


Out[28]:
LAT LONG DATE FAG100
9219 -53.113036 -70.933760 20141122 -11.20
9220 -53.113503 -70.933209 20141122 -11.34
9221 -53.113971 -70.932657 20141122 -11.48
9222 -53.114439 -70.932104 20141122 -11.62
9223 -53.114909 -70.931550 20141122 -11.76

In [35]:
df_confirmed_data['DATE'] = pd.to_datetime(df_confirmed_data['DATE'].apply(str), format='%Y%m%d')
df_confirmed_data['DATE'].head()


---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
c:\users\danielmateuspires\coding\icy-polar-opposites\venv\lib\site-packages\pandas\tseries\tools.py in _convert_listlike(arg, box, format, name, tz)
    409             try:
--> 410                 values, tz = tslib.datetime_to_datetime64(arg)
    411                 return DatetimeIndex._simple_new(values, name=name, tz=tz)

pandas\tslib.pyx in pandas.tslib.datetime_to_datetime64 (pandas\tslib.c:33358)()

TypeError: Unrecognized value type: <class 'str'>

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
<ipython-input-35-40e550062245> in <module>()
----> 1 df_confirmed_data['DATE'] = pd.to_datetime(df_confirmed_data['DATE'].apply(str), format='%Y%m%d')
      2 df_confirmed_data['DATE'].head()

c:\users\danielmateuspires\coding\icy-polar-opposites\venv\lib\site-packages\pandas\util\decorators.py in wrapper(*args, **kwargs)
     89                 else:
     90                     kwargs[new_arg_name] = new_arg_value
---> 91             return func(*args, **kwargs)
     92         return wrapper
     93     return _deprecate_kwarg

c:\users\danielmateuspires\coding\icy-polar-opposites\venv\lib\site-packages\pandas\tseries\tools.py in to_datetime(arg, errors, dayfirst, yearfirst, utc, box, format, exact, coerce, unit, infer_datetime_format)
    419     elif isinstance(arg, ABCSeries):
    420         from pandas import Series
--> 421         values = _convert_listlike(arg._values, False, format)
    422         return Series(values, index=arg.index, name=arg.name)
    423     elif isinstance(arg, (ABCDataFrame, MutableMapping)):

c:\users\danielmateuspires\coding\icy-polar-opposites\venv\lib\site-packages\pandas\tseries\tools.py in _convert_listlike(arg, box, format, name, tz)
    411                 return DatetimeIndex._simple_new(values, name=name, tz=tz)
    412             except (ValueError, TypeError):
--> 413                 raise e
    414 
    415     if arg is None:

c:\users\danielmateuspires\coding\icy-polar-opposites\venv\lib\site-packages\pandas\tseries\tools.py in _convert_listlike(arg, box, format, name, tz)
    378                     try:
    379                         result = tslib.array_strptime(arg, format, exact=exact,
--> 380                                                       errors=errors)
    381                     except tslib.OutOfBoundsDatetime:
    382                         if errors == 'raise':

pandas\tslib.pyx in pandas.tslib.array_strptime (pandas\tslib.c:62796)()

ValueError: time data '2014-11-22 00:00:00' does not match format '%Y%m%d' (match)

In [32]:
df_confirmed_data['DATE'] = df_confirmed_data['DATE'].apply(str)

In [33]:
df_confirmed_data.head()  # done


Out[33]:
LAT LONG DATE FAG100
9219 -53.113036 -70.933760 2014-11-22 00:00:00 -11.20
9220 -53.113503 -70.933209 2014-11-22 00:00:00 -11.34
9221 -53.113971 -70.932657 2014-11-22 00:00:00 -11.48
9222 -53.114439 -70.932104 2014-11-22 00:00:00 -11.62
9223 -53.114909 -70.931550 2014-11-22 00:00:00 -11.76

In [34]:
df_confirmed_data.head().to_json(orient = 'records')


Out[34]:
'[{"LAT":-53.1130361,"LONG":-70.9337602,"DATE":"2014-11-22 00:00:00","FAG100":-11.2},{"LAT":-53.1135029,"LONG":-70.9332092,"DATE":"2014-11-22 00:00:00","FAG100":-11.34},{"LAT":-53.1139706,"LONG":-70.9326573,"DATE":"2014-11-22 00:00:00","FAG100":-11.48},{"LAT":-53.1144394,"LONG":-70.9321043,"DATE":"2014-11-22 00:00:00","FAG100":-11.62},{"LAT":-53.1149094,"LONG":-70.9315501,"DATE":"2014-11-22 00:00:00","FAG100":-11.76}]'

In [ ]:
df_confirmed_data.to_json("../data/gravity_1.json",orient = 'records') # write to file

In [ ]: