Some imports. Overzealous, copied from a different notebook.



In [ ]:

    
%load_ext autoreload
%autoreload 2









    



The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload



In [ ]:

    
import pymarc
import random
from bookwormMARC.bookwormMARC import BRecord
from bookwormMARC.bookwormMARC import parse_record
from bookwormMARC.hathi_methods import All_Hathi
from bookwormMARC.bookwormMARC import LCCallNumber
import bz2
import bookwormMARC
import sys
import os
from collections import defaultdict


#all_files = hathi_record_yielder()
import pymarc
import ujson as json
import gzip

This converts a json object into a MARC class so the existing methods will work



In [ ]:

    
all_hathi = All_Hathi("/home/bschmidt/data/hathi_metadata/")

This class has the base names of the files and my directory structure hard-coded in: run elsewhere, you'll need to change that



In [ ]:

    
demo = []
for i,entry in enumerate(all_hathi):
    demo.append(entry)
    if i > 10:
        break
print demo









    



Reading fn
[{'literary_form': 'Not fiction', 'contributing_library': 'University of Michigan', 'permalink': u'https://babel.hathitrust.org/cgi/pt?id=mdp.39015018415946', 'serial_killer_guess': 'book', 'cataloging_source': u' ', 'scanner': u'google', 'language': u'slo', 'title': u'Slovensk\xe9 vyst\u0315ahovalectvo. /--- v.5', 'government_document': u' ', 'target_audience': 'Unknown or not specified', 'filename': u'mdp.39015018415946', 'cntry': u'cs ', 'rights_changed_date': u'2013-08-01', 'searchstring': u'<a href=https://babel.hathitrust.org/cgi/pt?id=mdp.39015018415946><em>Slovensk\xe9 vyst\u0315ahovalectvo. /--- v.5</em> (1969)', 'first_place': u'Bratislava, :', 'first_author_name': u'Bielik, Franti\u0161ek.', 'first_publisher': u'SAV, t. Svornost\u0315,', 'marc_record_created': u'1988-07-15', 'date': 1969, 'resource_type': 'book', 'record_date': 1969}, {'literary_form': 'Not fiction', 'contributing_library': 'University of Michigan', 'permalink': u'https://babel.hathitrust.org/cgi/pt?id=mdp.39015066356547', 'serial_killer_guess': 'book', 'cataloging_source': u' ', 'scanner': u'google', 'language': u'slo', 'title': u'Slovensk\xe9 vyst\u0315ahovalectvo. /--- v.1', 'government_document': u' ', 'target_audience': 'Unknown or not specified', 'filename': u'mdp.39015066356547', 'cntry': u'cs ', 'rights_changed_date': u'2013-08-01', 'searchstring': u'<a href=https://babel.hathitrust.org/cgi/pt?id=mdp.39015066356547><em>Slovensk\xe9 vyst\u0315ahovalectvo. /--- v.1</em> (1969)', 'first_place': u'Bratislava, :', 'first_author_name': u'Bielik, Franti\u0161ek.', 'first_publisher': u'SAV, t. Svornost\u0315,', 'marc_record_created': u'1988-07-15', 'date': 1969, 'resource_type': 'book', 'record_date': 1969}, {'literary_form': 'Not fiction', 'contributing_library': 'University of California', 'permalink': u'https://babel.hathitrust.org/cgi/pt?id=uc1.$b759626', 'serial_killer_guess': 'book', 'cataloging_source': u' ', 'scanner': u'google', 'language': u'slo', 'title': u'Slovensk\xe9 vyst\u0315ahovalectvo. /--- v. 1', 'government_document': u' ', 'target_audience': 'Unknown or not specified', 'filename': u'uc1.$b759626', 'cntry': u'cs ', 'rights_changed_date': u'2013-08-02', 'searchstring': u'<a href=https://babel.hathitrust.org/cgi/pt?id=uc1.$b759626><em>Slovensk\xe9 vyst\u0315ahovalectvo. /--- v. 1</em> (1969)', 'first_place': u'Bratislava, :', 'first_author_name': u'Bielik, Franti\u0161ek.', 'first_publisher': u'SAV, t. Svornost\u0315,', 'marc_record_created': u'1988-07-15', 'date': 1969, 'resource_type': 'book', 'record_date': 1969}, {'literary_form': 'Not fiction', 'contributing_library': 'University of California', 'permalink': u'https://babel.hathitrust.org/cgi/pt?id=uc1.$b759627', 'serial_killer_guess': 'book', 'cataloging_source': u' ', 'scanner': u'google', 'language': u'slo', 'title': u'Slovensk\xe9 vyst\u0315ahovalectvo. /--- v. 2', 'government_document': u' ', 'target_audience': 'Unknown or not specified', 'filename': u'uc1.$b759627', 'cntry': u'cs ', 'rights_changed_date': u'2013-08-02', 'searchstring': u'<a href=https://babel.hathitrust.org/cgi/pt?id=uc1.$b759627><em>Slovensk\xe9 vyst\u0315ahovalectvo. /--- v. 2</em> (1969)', 'first_place': u'Bratislava, :', 'first_author_name': u'Bielik, Franti\u0161ek.', 'first_publisher': u'SAV, t. Svornost\u0315,', 'marc_record_created': u'1988-07-15', 'date': 1969, 'resource_type': 'book', 'record_date': 1969}, {'literary_form': 'Not fiction', 'contributing_library': 'University of California', 'permalink': u'https://babel.hathitrust.org/cgi/pt?id=uc1.$b759628', 'serial_killer_guess': 'book', 'cataloging_source': u' ', 'scanner': u'google', 'language': u'slo', 'title': u'Slovensk\xe9 vyst\u0315ahovalectvo. /--- v. 3', 'government_document': u' ', 'target_audience': 'Unknown or not specified', 'filename': u'uc1.$b759628', 'cntry': u'cs ', 'rights_changed_date': u'2013-08-02', 'searchstring': u'<a href=https://babel.hathitrust.org/cgi/pt?id=uc1.$b759628><em>Slovensk\xe9 vyst\u0315ahovalectvo. /--- v. 3</em> (1969)', 'first_place': u'Bratislava, :', 'first_author_name': u'Bielik, Franti\u0161ek.', 'first_publisher': u'SAV, t. Svornost\u0315,', 'marc_record_created': u'1988-07-15', 'date': 1969, 'resource_type': 'book', 'record_date': 1969}, {'literary_form': 'Not fiction', 'contributing_library': 'University of Michigan', 'permalink': u'https://babel.hathitrust.org/cgi/pt?id=mdp.39015066356406', 'serial_killer_guess': 'book', 'cataloging_source': u' ', 'scanner': u'google', 'language': u'slo', 'title': u'Slovensk\xe9 vyst\u0315ahovalectvo. /--- v.2', 'government_document': u' ', 'target_audience': 'Unknown or not specified', 'filename': u'mdp.39015066356406', 'cntry': u'cs ', 'rights_changed_date': u'2013-08-01', 'searchstring': u'<a href=https://babel.hathitrust.org/cgi/pt?id=mdp.39015066356406><em>Slovensk\xe9 vyst\u0315ahovalectvo. /--- v.2</em> (1969)', 'first_place': u'Bratislava, :', 'first_author_name': u'Bielik, Franti\u0161ek.', 'first_publisher': u'SAV, t. Svornost\u0315,', 'marc_record_created': u'1988-07-15', 'date': 1969, 'resource_type': 'book', 'record_date': 1969}, {'literary_form': 'Not fiction', 'contributing_library': 'University of Michigan', 'permalink': u'https://babel.hathitrust.org/cgi/pt?id=mdp.39015066356695', 'serial_killer_guess': 'book', 'cataloging_source': u' ', 'scanner': u'google', 'language': u'slo', 'title': u'Slovensk\xe9 vyst\u0315ahovalectvo. /--- v.3', 'government_document': u' ', 'target_audience': 'Unknown or not specified', 'filename': u'mdp.39015066356695', 'cntry': u'cs ', 'rights_changed_date': u'2013-08-01', 'searchstring': u'<a href=https://babel.hathitrust.org/cgi/pt?id=mdp.39015066356695><em>Slovensk\xe9 vyst\u0315ahovalectvo. /--- v.3</em> (1969)', 'first_place': u'Bratislava, :', 'first_author_name': u'Bielik, Franti\u0161ek.', 'first_publisher': u'SAV, t. Svornost\u0315,', 'marc_record_created': u'1988-07-15', 'date': 1969, 'resource_type': 'book', 'record_date': 1969}, {'literary_form': 'Not fiction', 'contributing_library': 'University of Michigan', 'permalink': u'https://babel.hathitrust.org/cgi/pt?id=mdp.39015066356554', 'serial_killer_guess': 'book', 'cataloging_source': u' ', 'scanner': u'google', 'language': u'slo', 'title': u'Slovensk\xe9 vyst\u0315ahovalectvo. /--- v.4', 'government_document': u' ', 'target_audience': 'Unknown or not specified', 'filename': u'mdp.39015066356554', 'cntry': u'cs ', 'rights_changed_date': u'2013-08-01', 'searchstring': u'<a href=https://babel.hathitrust.org/cgi/pt?id=mdp.39015066356554><em>Slovensk\xe9 vyst\u0315ahovalectvo. /--- v.4</em> (1969)', 'first_place': u'Bratislava, :', 'first_author_name': u'Bielik, Franti\u0161ek.', 'first_publisher': u'SAV, t. Svornost\u0315,', 'marc_record_created': u'1988-07-15', 'date': 1969, 'resource_type': 'book', 'record_date': 1969}, {'cataloging_source': u'd', 'scanner': u'google', 'first_publisher': u'Departemen Pendidikan dan Kebudayaan, Proyek Penerbitan Buku Sastra Indonesia dan Daerah,', 'item_date': 1981, 'rights_changed_date': u'2013-08-01', 'literary_form': 'Not fiction', 'serial_killer_guess': 'book', 'title': u'Burangir na hombang / disusun oleh Ch. St. Tinggibarani Perkasa Alam [dan] Dra. Rukiyah Siregar.', 'filename': u'mdp.39015033913115', 'contributing_library': 'University of Michigan', 'searchstring': u'<a href=https://babel.hathitrust.org/cgi/pt?id=mdp.39015033913115><em>Burangir na hombang / disusun oleh Ch. St. Tinggibarani Perkasa Alam [dan] Dra. Rukiyah Siregar.</em> (1981)', 'target_audience': 'Unknown or not specified', 'cntry': u'io ', 'first_place': u'Jakarta :', 'date': 1981, 'permalink': u'https://babel.hathitrust.org/cgi/pt?id=mdp.39015033913115', 'language': u'ind', 'government_document': u' ', 'subject_places': [u'a-io---'], 'record_date': 1981, 'marc_record_created': u'1988-07-15', 'resource_type': 'book'}, {'cataloging_source': u' ', 'scanner': u'google', 'lc0': 'Q', 'lc1': 'QA', 'date': 1969, 'first_author_birth': 1924, 'item_date': 1969, 'rights_changed_date': u'2013-08-01', 'lc2': '935', 'literary_form': 'Not fiction', 'serial_killer_guess': 'book', 'title': u'Free vibrations of circular cylindrical shells,', 'filename': u'mdp.39015000995467', 'first_author_name': u'Armen\xe0kas, Anthony E., 1924-', 'contributing_library': 'University of Michigan', 'searchstring': u'<a href=https://babel.hathitrust.org/cgi/pt?id=mdp.39015000995467><em>Free vibrations of circular cylindrical shells,</em> (1969)', 'target_audience': 'Unknown or not specified', 'cntry': u'enk', 'first_place': u'Oxford,', 'lc_class_from_lc': True, 'first_publisher': u'Pergamon Press', 'permalink': u'https://babel.hathitrust.org/cgi/pt?id=mdp.39015000995467', 'language': u'eng', 'government_document': u'|', 'record_date': 1969, 'marc_record_created': u'1988-07-15', 'resource_type': 'book'}, {'cataloging_source': u' ', 'scanner': u'google', 'lc0': 'E', 'lc1': 'E', 'date': 1969, 'first_author_birth': 1923, 'item_date': 1969, 'rights_changed_date': u'2013-08-01', 'lc2': '185', 'literary_form': 'Not fiction', 'serial_killer_guess': 'book', 'title': u'The making of black America; essays in Negro life & history,--- v.1', 'filename': u'mdp.39015002382565', 'first_author_name': u'Meier, August, 1923-2003.', 'contributing_library': 'University of Michigan', 'searchstring': u'<a href=https://babel.hathitrust.org/cgi/pt?id=mdp.39015002382565><em>The making of black America; essays in Negro life & history,--- v.1</em> (1969)', 'target_audience': 'Unknown or not specified', 'first_author_death': 2003, 'cntry': u'nyu', 'first_place': u'New York,', 'lc_class_from_lc': True, 'first_publisher': u'Atheneum,', 'permalink': u'https://babel.hathitrust.org/cgi/pt?id=mdp.39015002382565', 'language': u'eng', 'government_document': u'|', 'record_date': 1969, 'marc_record_created': u'1988-07-15', 'resource_type': 'book'}, {'cataloging_source': u' ', 'scanner': u'google', 'lc0': 'E', 'lc1': 'E', 'date': 1969, 'first_author_birth': 1923, 'item_date': 1969, 'rights_changed_date': u'2013-08-01', 'lc2': '185', 'literary_form': 'Not fiction', 'serial_killer_guess': 'book', 'title': u'The making of black America; essays in Negro life & history,--- v.2', 'filename': u'mdp.39015002379926', 'first_author_name': u'Meier, August, 1923-2003.', 'contributing_library': 'University of Michigan', 'searchstring': u'<a href=https://babel.hathitrust.org/cgi/pt?id=mdp.39015002379926><em>The making of black America; essays in Negro life & history,--- v.2</em> (1969)', 'target_audience': 'Unknown or not specified', 'first_author_death': 2003, 'cntry': u'nyu', 'first_place': u'New York,', 'lc_class_from_lc': True, 'first_publisher': u'Atheneum,', 'permalink': u'https://babel.hathitrust.org/cgi/pt?id=mdp.39015002379926', 'language': u'eng', 'government_document': u'|', 'record_date': 1969, 'marc_record_created': u'1988-07-15', 'resource_type': 'book'}]

Here's where we write it out.



In [ ]:

    
if __name__=="__main__":
    all_hathi = All_Hathi("/home/bschmidt/data/hathi_metadata/")
    dump = gzip.open("~/hathi_metadata/jsoncatalog_full.txt.gz","w")
    for i,vol in enumerate(all_hathi):
        if i % 250000 == 0:
            sys.stdout.write("Reading item no. " + str(i) + "\n")
        dump.write(json.dumps(vol) + "\n")



In [ ]:



In [ ]: