In [1]:

    

import re, sys, urllib.request, time
import lxml.etree as ET


    

In [2]:

    

url = 'http://thun-korrespondenz.uibk.ac.at:8080/exist/rest/db/files/thun/xml'


    

In [3]:

    

try:
    u = urllib.request.urlopen(url)
except:
    print('something is wrong with this url')
    sys.exit(0)


    

In [4]:

    

dom = ET.parse(u)


    

In [5]:

    

resource = dom.xpath("//exist:resource/@name",
                    namespaces={'exist':
                               'http://exist.sourceforge.net/NS/exist'})


    

In [9]:

    

for file in resource:
    fileUrl = url+"/"+file
    try:
        u = urllib.request.urlopen(fileUrl)
    except:
        print('something is wrong with this url')
        sys.exit(0)
    text = u.read().decode(u.headers.get_content_charset())
    #print(text)
    notePattern = re.compile(r'<note.*?</note>', re.DOTALL)
    text = re.sub(notePattern, "", text)
    pattern = re.compile(r'<.*</index>', re.DOTALL)
    text = re.sub(pattern," ", text)
    pattern = re.compile(r"<.*?>", re.DOTALL)
    text = re.sub(pattern," ", text)
    text = text.lower()
    #print(text)
    fileName = re.sub(r'.xml', ".txt", file)
    if len(text) > 1000:
        saveName = "../data/"+fileName
        output = open(saveName, 'w', encoding="utf")
        output.write(text)
        output.close()
    else:
        print(fileName)
    time.sleep(1)


    

    




tomek-an-thun_1854-05-21_A3-XXI-D260.txt
thun-an-miklosich_1857-03-10_CA801.txt
thun-an-helfert_1854-06-18_CA1421.txt
zeitungsausschnitt_o.D._A3-XXI-D123.txt
philippsberg-an-thun-friedrich_1851-11-14_A3-XXI-D129a.txt
thun-an-schulte_09-29_CA504.txt
thun-an-fessler_oD_CA372.txt
thun-an-ficker_1860-10-20_CA190.txt
karte-oesterreich-finanzkraft-von-hornstein_1851_A3-XXI-D148.txt
thun-an-helfert_1854-03-07_CA1420.txt
schwarzenberg-an-thun_o.D._A3-XXI-D139.txt
thun-an-helfert_1851-03-20_CA1411.txt
thun-an-helfert_1851-04-08_CA1413.txt
thun-an-helfert_1852-02-08_CA1414.txt
thun-an-helfert_1853-08-21_CA1417.txt
mozart-an-thun_o.D._A3-XXI-D182.txt
thun-an-helfert_1860-02-14_CA1423.txt
thun-an-helfert_1860-07-02_CA1424.txt
thun-an-helfert_1855-02-12_CA1426.txt
thun-an-helfert_1859-02-09_CA1422.txt
thun-an-miklosich_02-01_CA799.txt
thun-an-exner_oD_CA82.txt
thun-an-exner_1852-08-31_CA80.txt
thun-an-laube_1858-09-22_CA820.txt
jelacic-an-caroline-thun_1850-03-30_A3-XXI-D37.txt
auszug-publikation-von-papiu-ilarian_1853-01-04_A3-XXI-D250.txt
bemerkungen-zu-gesetzentwurf-ueber-kirchliche-geb%C3%A4ude_1854-05_A3-XXI-D261.txt
stellungnahme-clam-handeslrecht_1854-07-07_A3-XXI-D265.txt
keller-an-thun_1854-10-15_A3-XXI-D293.txt
bernhard-an-thun_1854-09-15_A3-XXI-D283.txt
hirtenbrief-von-wierzchleyski_1854-11-08_A3-XXI-D297.txt
hirtenbrief-von-haynald_1854-11-22_A3-XXI-D301.txt
huebner-an-thun_oD_A3-XXI-D312.txt
metternich-an-thun_1855-01-31_A3-XXI-D326.txt

In [ ]: