notebook.community

Edit and run 





In [1]:



    


url = "https://nspires.nasaprs.com/external/viewrepositorydocument/cmdocumentid=492458/solicitationId=%7B68C12087-132D-3814-9A87-5323BCE6CAB6%7D/viewSolicitationDocument=1/Table%202%202016_amend8_clarify.html"



    











In [2]:



    


import requests



    











In [3]:



    


from bs4 import BeautifulSoup



    











In [4]:



    


r = requests.get(url)



    











In [5]:



    


soup = BeautifulSoup(r.text, 'lxml')



    











In [6]:



    


table = soup.find('table')



    











In [7]:



    


rows = table.find_all('tr')



    











In [8]:



    


def parse_header(row):
    headers = [i.get_text() for i in row.find_all('th')]
    headers = [i.strip() for i in headers]
    return [header.splitlines()[0].strip() for header in headers]



    











In [9]:



    


columns = parse_header(rows[0])
columns.append('url')



    











In [10]:



    


def parse_out_url(row):
    cells = row.find_all('td')
    cell = cells[1]
    a = cell.find('a')
    a['href'] = a['href'].strip().replace('http','https')
    return a

def parse_table(rows):
    cell_data = []
    for row in rows:
        cells = [cell.get_text().strip() for cell in row.find_all('td')]
        if len(cells) == 3:
            cells.append('None')
        cells.append(parse_out_url(row))
        cell_data.append(cells)
    return cell_data



    











In [11]:



    


df = pd.DataFrame(parse_table(rows[1:]), columns=columns)



    











In [12]:



    


from IPython.display import HTML



    











In [13]:



    


pd.set_option('display.max_colwidth', 1000)



    











In [14]:



    


HTML(df.to_html(escape=False))



    


















    
Out[14]:








  
    

      
      APPENDIX
      PROGRAM
      NOI/Step-1
      PROPOSAL
      url
    

  
  
    

      0
      B.4
      Heliophysics Guest Investigators
      03/18/2016 (Step-1)
      04/22/2016(Step-2)
      Heliophysics Guest Investigators
    

    

      1
      A.29
      NASA Data for Operation and Assessment
      03/15/2016
      05/20/2016
      NASA Data for Operation and Assessment
    

    

      2
      D.2
      Astrophysics Data Analysis
      03/25/2016
      05/13/2016
      Astrophysics Data Analysis
    

    

      3
      A.46
      Earth Science Applications: Ecological Forecasting
      03/16/2016
      05/26/2016
      Earth Science Applications: Ecological Forecasting
    

    

      4
      E.3
      Exoplanets Research Program [3]
      03/29/2016 (Step-1)
      05/26/2016(Step-2)
      Exoplanets Research Program [3]
    

    

      5
      A.11
      Ocean Surface Topography Science Team
      04/29/2016
      05/27/2016
      Ocean Surface Topography Science Team
    

    

      6
      C.2
      Emerging Worlds [3] [4]
      03/31/2016(Step-1)
      06/03/2016(Step-2)
      Emerging Worlds [3] [4]
    

    

      7
      A.31
      Utilization of Airborne Visible/Infrared Imaging Spectrometer - Next Generation Data from an Airborne Campaign in India
      04/07/2016
      06/10/2016
      Utilization of Airborne Visible/Infrared Imaging Spectrometer - Next Generation Data from an Airborne Campaign in India
    

    

      8
      C.6
      Solar System Observations [3] [4]
      04/08/2016(Step-1)
      06/10/2016(Step-2)
      Solar System Observations [3] [4]
    

    

      9
      A.24
      Earth Surface and Interior
      04/15/2016
      06/15/2016
      Earth Surface and Interior
    

    

      10
      A.5
      Carbon Cycle Science
      04/01/2016
      06/15/2016
      Carbon Cycle Science
    

    

      11
      C.10
      Cassini Data Analysis Program [3]
      04/06/2016(Step-1)
      06/16/2016(Step-2)
      Cassini Data Analysis Program [3]
    

    

      12
      A.13
      Modeling, Analysis, and Prediction
      04/15/2016
      06/17/2016
      Modeling, Analysis, and Prediction
    

    

      13
      C.18
      Laboratory Analysis of Returned Samples [3] [4]
      05/02/2016(Step-1)
      06/24/2016(Step-2)
      Laboratory Analysis of Returned Samples [3] [4]
    

    

      14
      A.8
      Physical Oceanography
      05/20/2016
      06/30/2016
      Physical Oceanography
    

    

      15
      A.17
      Atmospheric Composition: Upper Atmospheric Composition Observations
      N/A
      07/01/2016
      Atmospheric Composition: Upper Atmospheric Composition Observations
    

    

      16
      D.4
      Astrophysics Theory
      05/16/2016
      07/08/2016
      Astrophysics Theory
    

    

      17
      A.42
      Instrument Incubator Program
      05/31/2016
      07/11/2016
      Instrument Incubator Program
    

    

      18
      A.21
      Terrestrial Hydrology
      05/13/2016
      07/15/2016
      Terrestrial Hydrology
    

    

      19
      C.7
      Planetary Data Archiving, Restoration, and Tools
      05/13/2016(Step-1)
      07/15/2016(Step-2)
      Planetary Data Archiving, Restoration, and Tools
    

    

      20
      A.47
      Citizen Science for Earth Systems
      05/27/2016
      07/21/2016
      Citizen Science for Earth Systems
    

    

      21
      C.13
      Maturation of Instruments for Solar System Exploration [3]
      05/20/2016(Step-1)
      07/21/2016(Step-2)
      Maturation of Instruments for Solar System Exploration [3]
    

    

      22
      B.3
      Heliophysics Technology and Instrument Development for Science
      06/10/2016 (Step-1)
      07/22/2016(Step-2)
      Heliophysics Technology and Instrument Development for Science
    

    

      23
      B.7
      Heliophysics Data Environment Enhancements
      05/20/2016 (Step-1)
      07/22/2016(Step-2)
      Heliophysics Data Environment Enhancements
    

    

      24
      C.5
      Exobiology [3] [4]
      05/20/2016(Step-1)
      07/22/2016(Step-2)
      Exobiology [3] [4]
    

    

      25
      A.4
      Terrestrial Ecology
      05/16/2016
      08/01/2016
      Terrestrial Ecology
    

    

      26
      A.16
      Studies with IceSat and CryoSat-2
      N/A
      08/05/2016
      Studies with IceSat and CryoSat-2
    

    

      27
      A.19
      Atmospheric Composition: Aura Science Team and Atmospheric Composition Modeling and Analysis Program
      N/A
      08/17/2016
      Atmospheric Composition: Aura Science Team and Atmospheric Composition Modeling and Analysis Program
    

    

      28
      A.27
      Earth Science U.S. Participating Investigator
      N/A
      08/26/2016
      Earth Science U.S. Participating Investigator
    

    

      29
      A.37
      Applied Sciences - Water Resources
      05/02/2016 (Step-1)
      09/01/2016(Step-2)
      Applied Sciences - Water Resources
    

    

      30
      C.15
      Planetary Protection Research [4]
      06/24/2016
      09/02/2016
      Planetary Protection Research [4]
    

    

      31
      B.2
      Heliophysics Supporting Research
      07/29/2016 (Step-1)
      09/09/2016(Step-2)
      Heliophysics Supporting Research
    

    

      32
      A.18
      Cloud and Aerosol Monsoonal Processes - Philippines Experiment
      N/A
      09/23/2016
      Cloud and Aerosol Monsoonal Processes - Philippines Experiment
    

    

      33
      C.14
      Planetary Science and Technology Through Analog Research [3] [4]
      07/22/2016(Step-1)
      09/23/2016(Step-2)
      Planetary Science and Technology Through Analog Research [3] [4]
    

    

      34
      D.5
      Swift Guest Investigator - Cycle 13
      N/A
      09/23/2016
      Swift Guest Investigator - Cycle 13
    

    

      35
      A.26
      Airborne Instrument Technology Transition
      07/25/2016
      09/26/2016
      Airborne Instrument Technology Transition
    

    

      36
      A.28
      Interdisciplinary Science
      08/01/2016
      09/29/2016
      Interdisciplinary Science
    

    

      37
      A.9
      Ocean Salinity Science Team
      09/30/2016
      10/28/2016
      Ocean Salinity Science Team
    

    

      38
      C.8
      Lunar Data Analysis [3]
      08/26/2016(Step-1)
      10/28/2016(Step-2)
      Lunar Data Analysis [3]
    

    

      39
      C.9
      Mars Data Analysis [3]
      08/26/2016(Step-1)
      10/28/2016(Step-2)
      Mars Data Analysis [3]
    

    

      40
      B.5
      Heliophysics Grand Challenges Research
      09/23/2016 (Step-1)
      11/04/2016(Step-2)
      Heliophysics Grand Challenges Research
    

    

      41
      D.7
      K2 Guest Observer - Cycle 5
      09/23/2016 (Step-1)
      11/04/2016(Step-2)
      K2 Guest Observer - Cycle 5
    

    

      42
      C.12
      Planetary Instrument Concepts for the Advancement of Solar System Observations [3]
      09/14/2016(Step-1)
      11/14/2016(Step-2)
      Planetary Instrument Concepts for the Advancement of Solar System Observations [3]
    

    

      43
      A.10
      Sea Level Change Science Team
      10/14/2016
      11/15/2016
      Sea Level Change Science Team
    

    

      44
      C.11
      Discovery Data Analysis [3]
      09/08/2016(Step-1)
      11/17/2016(Step-2)
      Discovery Data Analysis [3]
    

    

      45
      B.6
      Heliophysics Living With a Star Science
      10/07/2016 (Step-1)
      11/18/2016(Step-2)
      Heliophysics Living With a Star Science
    

    

      46
      D.10
      NuSTAR Guest Observer - Cycle 3
      N/A
      01/13/2017
      NuSTAR Guest Observer - Cycle 3
    

    

      47
      D.6
      Fermi Guest Investigator - Cycle 10
      N/A
      01/20/2017
      Fermi Guest Investigator - Cycle 10
    

    

      48
      E.4
      Habitable Worlds [3] [4]
      11/18/2016 (Step-1)
      01/20/2017(Step-2)
      Habitable Worlds [3] [4]
    

    

      49
      C.3
      Solar System Workings [3] [4]
      11/17/2016(Step-1)
      02/23/2017(Step-2)
      Solar System Workings [3] [4]
    

    

      50
      D.3
      Astrophysics Research and Analysis
      01/20/2017
      03/17/2017
      Astrophysics Research and Analysis
    

    

      51
      D.8
      Strategic Astrophysics Technology
      01/20/2017
      03/17/2017
      Strategic Astrophysics Technology
    

    

      52
      A.25
      Rapid Response and Novel Research in Earth Science
      N/A
      Rolling Submissions through 03/31/2017
      Rapid Response and Novel Research in Earth Science
    

    

      53
      C.16
      Fellowships for Early Career Researchers (current fellows) [3]
      N/A
      Rolling Submissions through 03/31/2017
      Fellowships for Early Career Researchers (current fellows) [3]
    

    

      54
      E.2
      Topical Workshops, Symposia, and Conferences
      N/A
      Rolling Submissions through 03/31/2017
      Topical Workshops, Symposia, and Conferences
    

    

      55
      A.2
      Land Cover/Land Use Change
      12/01/2016 (Step 1)
      06/01/2017 (Step 2)
      Land Cover/Land Use Change
    

    

      56
      C.16
      Fellowships for Early Career Researchers (new applicants) [3]
      See Program of Interest
      None
      Fellowships for Early Career Researchers (new applicants) [3]
    

    

      57
      C.17
      Planetary Major Equipment [4]
      See Program of Interest
      None
      Planetary Major Equipment [4]
    

    

      58
      A.3
      Ocean Biology and Biogeochemistry
      TBD
      TBD
      Ocean Biology and Biogeochemistry
    

    

      59
      A.7
      Carbon Monitoring System
      TBD
      TBD
      Carbon Monitoring System
    

    

      60
      A.14
      Cryospheric Science
      TBD
      TBD
      Cryospheric Science
    

    

      61
      A.23
      Weather and Atmospheric Dynamics
      TBD
      TBD
      Weather and Atmospheric Dynamics
    

    

      62
      A.30
      Remote Sensing of Water Quality
      TBD
      TBD
      Remote Sensing of Water Quality
    

    

      63
      A.41
      Advanced Information Systems Technology
      TBD
      TBD
      Advanced Information Systems Technology
    

    

      64
      B.8
      Magnetospheric Multiscale Guest Investigators
      TBD
      TBD
      Magnetospheric Multiscale Guest Investigators
    

    

      65
      B.9
      Heliophysics Grand Challenges Research - Science Centers
      TBD
      TBD
      Heliophysics Grand Challenges Research - Science Centers
    

    

      66
      B.10
      Heliophysics U.S. Principal Investigator
      TBD
      TBD
      Heliophysics U.S. Principal Investigator
    

    

      67
      C.19
      New Frontiers Data Analysis Program [3]
      TBD
      TBD
      New Frontiers Data Analysis Program [3]
    

    

      68
      C.20
      Concepts for Ocean worlds Life Detection Technology
      TBD
      TBD
      Concepts for Ocean worlds Life Detection Technology
    

    

      69
      D.11
      ASTRO-H Guest Observer - Cycle 1
      TBD
      TBD
      ASTRO-H Guest Observer - Cycle 1
    

    

      70
      A.6
      Biodiversity
      Not solicited this year
      None
      Biodiversity
    

    

      71
      A.12
      Ocean Vector Winds Science Team
      Not solicited this year
      None
      Ocean Vector Winds Science Team
    

    

      72
      A.15
      IceBridge Observations
      Not solicited this year
      None
      IceBridge Observations
    

    

      73
      A.20
      Atmospheric Composition: Tropospheric Composition Program
      Not solicited this year
      None
      Atmospheric Composition: Tropospheric Composition Program
    

    

      74
      A.22
      NASA Energy and Water Cycle
      Not solicited this year
      None
      NASA Energy and Water Cycle
    

    

      75
      A.32
      New Investigator Program
      Not solicited this year
      None
      New Investigator Program
    

    

      76
      A.33
      SUOMI National Polar-Orbiting Partnership (NPP) Science Team and Science Investigator-Led Processing Systems for Earth System Data Records from SUOMI NPP
      Not solicited this year
      None
      SUOMI National Polar-Orbiting Partnership (NPP) Science Team and Science Investigator-Led Processing Systems for Earth System Data Records from SUOMI NPP
    

    

      77
      A.34
      Science of Terra & Aqua
      Not solicited this year
      None
      Science of Terra & Aqua
    

    

      78
      A.35
      Terra and Aqua Existing Algorithms
      Not solicited this year
      None
      Terra and Aqua Existing Algorithms
    

    

      79
      A.36
      PACE Science Team
      Not solicited this year
      None
      PACE Science Team
    

    

      80
      A.38
      Advancing Collaborative Connections for Earth System Science
      Not solicited this year
      None
      Advancing Collaborative Connections for Earth System Science
    

    

      81
      A.39
      Making Earth System Data Records for Use in Research Environments
      Not solicited this year
      None
      Making Earth System Data Records for Use in Research Environments
    

    

      82
      A.40
      Computational Modeling Algorithms and Cyberinfrastructure
      Not solicited this year
      None
      Computational Modeling Algorithms and Cyberinfrastructure 
    

    

      83
      A.43
      Advanced Component Technology
      Not solicited this year
      None
      Advanced Component Technology 
    

    

      84
      A.44
      In-Space Validation of Earth Science Technologies
      Not solicited this year
      None
      In-Space Validation of Earth Science Technologies
    

    

      85
      A.45
      Sustainable Land Imaging Technology
      Not solicited this year
      None
      Sustainable Land Imaging Technology
    

    

      86
      D.9
      Nancy Grace Roman Technology Fellowships for Early Career Researchers
      Not solicited this year
      None
      Nancy Grace Roman Technology Fellowships for Early Career Researchers
    

    

      87
      A.1
      Earth Science Research Program Overview
      N/A
      N/A
      Earth Science Research Program Overview
    

    

      88
      B.1
      Heliophysics Research Program Overview
      N/A
      N/A
      Heliophysics Research Program Overview
    

    

      89
      C.1
      Planetary Science Research Program Overview
      N/A
      N/A
      Planetary Science Research Program Overview
    

    

      90
      D.1
      Astrophysics Research Program Overview
      N/A
      N/A
      Astrophysics Research Program Overview
    

    

      91
      E.1
      Cross Division Research Overview
      N/A
      N/A
      Cross Division Research Overview
    

  


















In [15]:



    


def parse_date_cell(cell):
    try:
        tokens = cell.split('(')
    except AttributeError:
        return cell, False
    date = tokens[0]
    step = True if len(tokens)==2 else False        
    if date == 'N/A':
        d = pd.NaT
    elif date[0].isdigit():
        t = pd.to_datetime(str(date))
        d = t.date()
    else:
        d = date
    return d, step
    
def parse_date_columns(row):
    cell = row['NOI/Step-1']
    date1, step1 = parse_date_cell(cell)
    cell = row['PROPOSAL']
    date2, step2 = parse_date_cell(cell)
    if step1!=step2:
        raise ValueError('steps not the same')
    return pd.Series([date1, step1, date2], index=['date1', 'step', 'date2'])



    











In [16]:



    


df = pd.concat([df, df.apply(parse_date_columns, axis=1)], axis=1)



    











In [18]:



    


from icalendar import Calendar, Event



    











In [19]:



    


cal = Calendar()

cal.add('prodid', '-//NASA ROSES deadline calendar//mxm.dk//')
cal.add('version', '2.0')
cal.add('name', 'NASA ROSES Deadlines')
cal.add('x-wr-calname', 'NASA ROSES Deadlines')
cal.add('x-wr-caldesc', 'NASA ROSES Deadlines for 2016')



    











In [20]:



    


def create_cal_events_from_row(row, i):
    events = []
    for j,col in enumerate(['date1', 'date2']):
        date = row[col]

        event = Event()
        event.add('dtstart', date)

        descr = 'APPENDIX '
        descr += row['APPENDIX'] + '\n'
        event.add('description', descr)

        sumtext = 'ROSES D/L '
        if row['step']:
            if col == 'date1':
                sumtext += 'Step_1 '
            else:
                sumtext += 'Step_2 '
        else:
            if col == 'date':
                sumtext += 'NOI '
            else:
                sumtext += 'Final '

        event.add('summary', sumtext + row['PROGRAM'])
        s = row['url'].get('href')
        event.add('location', s)
        s = s.replace('&', '%26')
        s = s.replace('?', '%3F')
        s = s.replace('{', '%7B')
        s = s.replace('}', '%7D')
        event.add('url', s)
        uid = 'nasa_roses_deadlines_2016_'+str(i)+str(j)
        event.add('uid', uid)
        events.append(event)
    return events



    











In [21]:



    


import datetime
for i, row in df.iterrows():
    date = row['date1']
    if type(date) == datetime.date:
        for event in create_cal_events_from_row(row, i):
            cal.add_component(event)



    











In [22]:



    


import os

with open(os.path.join(os.environ['HOME'], 'NASA_ROSES_deadlines.ics'), 'wb') as f:
    f.write(cal.to_ical())

!open {f.name}

Content source: michaelaye/planetpy

Similar notebooks:

notebook.community | gallery | about