In [1]:

    

from bs4 import BeautifulSoup
import os
import glob


    

In [4]:

    

i = 0
with open("all_packages_metadata.csv", "w") as fo:
    fo.write(";".join(["Package", 
                          "Version:",
                          'Depends:',
                          'Imports:',
                          'Published:',
                          'Author:',
                          'Maintainer:',
                          'License:']) + "\n")
    for myfile in glob.glob("raw_html/*.html"):
        with open(myfile) as f:
            html = f.read()
        html = BeautifulSoup(html, "lxml")
        table = html.find_all('table')[0]
        values = ['NA'] * 8
        values[0] = os.path.basename(myfile)[:-5]
        for row in table.find_all('tr'):
            columns = row.find_all('td')
            tmp = []
            for column in columns:
                tmp.append(column.get_text().replace("\n", " "))
            if tmp[0] == "Version:":
                values[1] = tmp[1]
            elif tmp[0] == "Depends:":
                values[2] = tmp[1]
            elif tmp[0] == "Imports:":
                values[3] = tmp[1]
            elif tmp[0] == "Published:":
                values[4] = tmp[1]
            elif tmp[0] == "Author:":
                values[5] = tmp[1]
            elif tmp[0] == "Maintainer:":
                values[6] = tmp[1]
            elif tmp[0] == "License:":
                values[7] = tmp[1]
        fo.write(";".join(values) + "\n")
        if i % 100 == 0:
            print(i, values[0])
        i = i + 1


    

    




0 bitrugs
100 SensitivityCaseControl
200 msgps
300 ABHgenotypeR
400 signmedian.test
500 SMC
600 vembedr
700 SurvRegCensCov
800 pulsar
900 GLMaSPU
1000 seeclickfixr
1100 capwire
1200 probFDA
1300 entropy
1400 INSPIRE
1500 reporttools
1600 biogeo
1700 odbc
1800 gWidgets2
1900 cg
2000 SetRank
2100 ExceedanceTools
2200 texmex
2300 structSSI
2400 rprev
2500 PurBayes
2600 cudaBayesreg
2700 robotstxt
2800 pems.utils
2900 kinfit
3000 aster
3100 RcmdrPlugin.FactoMineR
3200 vrmlgen
3300 labstats
3400 fishmove
3500 bpp
3600 ddR
3700 kineticF
3800 janitor
3900 MHTdiscrete
4000 orgR
4100 matpow
4200 Stem
4300 gemtc
4400 micEconIndex
4500 softmaxreg
4600 ProbForecastGOP
4700 MAMS
4800 FCNN4R
4900 tumgr
5000 frailtypack
5100 sp
5200 dropR
5300 rcdk
5400 bnspatial
5500 TTmoment
5600 TRSbook
5700 AmostraBrasil
5800 httpuv
5900 JSM
6000 RobustGaSP
6100 TH.data
6200 shinytoastr
6300 GenABEL
6400 metavcov
6500 DEMEtics
6600 spatialEco
6700 compare
6800 equate
6900 GibbsACOV
7000 Counterfactual
7100 useful
7200 astrodatR
7300 tab
7400 tsBSS
7500 RepeatABEL
7600 joint.Cox
7700 matie
7800 fuzzywuzzyR
7900 WRTDStidal
8000 jmotif
8100 hyperSMURF
8200 dendextend
8300 BGLR
8400 rMR
8500 ClustOfVar
8600 qualV
8700 poisson.glm.mix
8800 plotrix
8900 chords
9000 seqMeta
9100 fracprolif
9200 PANDA
9300 Deducer
9400 VizOR
9500 spTDyn
9600 genMOSS
9700 AMAP.Seq
9800 PBImisc
9900 fICA
10000 ergm.count
10100 anim.plots
10200 FastHCS
10300 ddpcr
10400 forestplot

In [ ]: