In [1]:

    

from __future__ import print_function


    

In [2]:

    

import os
import json
import shutil
import sh
import yaml
from pandas import DataFrame, Series
from itertools import islice

REPOS_LIST = "/Users/raymondyee/C/src/gitenberg/Second-Folio/list_of_repos.txt"
GITENBERG_DIR = "/Users/raymondyee/C/src/gitenberg/"

METADATA_DIR = "/Users/raymondyee/C/src/gitenberg-dev/giten_site/metadata"
COVERS_DATA = "/Users/raymondyee/C/src/gitenberg/Second-Folio/covers_data.json"


    

In [3]:

    

import os
import glob
import sh
import yaml

from gitenberg import metadata
import jinja2

from second_folio import (GITENBERG_DIR, 
                          all_repos, 
                          apply_to_repos, 
                          travis_template, 
                          travis_setup_releases, 
                          git_pull,
                          apply_travis,
                          finish_travis,
                          repo_is_buildable,
                          has_travis_with_gitenberg_build,
                          slugify,
                          write_repo_token_file,
                          latest_epub,
                          repo_version
                          )

from github_settings import (username, password)


    

In [4]:

    

from itertools import islice, izip

# pick subset of repositories to calculate on
repos = list(islice(all_repos,0,None))

# determine which repos are "buildable"
repos_statues = list(izip(repos, 
                          apply_to_repos(repo_is_buildable, repos=repos), 
                          apply_to_repos(has_travis_with_gitenberg_build, repos=repos) ))

# we want to apply travis to repos that are buildable but that don't yet have .travis.yml. 

repos_to_travisfy = [repo[0] for repo in repos_statues if repo[1] and not repo[2]]
repos_to_travisfy


    

    
Out[4]:





[]

In [5]:

    

all_repos


    

    
Out[5]:





['Adventures-of-Huckleberry-Finn_76',
 'Don-Quixote_996',
 'Dubliners_2814',
 'Jane-Eyre_1260',
 'Moby-Dick--Or-The-Whale_2701',
 'Narrative-of-the-Life-of-Frederick-Douglass-an-American-Slave_23',
 'Pride-and-Prejudice_1342',
 'The-Adventures-of-Sherlock-Holmes_1661',
 'The-Brothers-Karamazov_28054',
 'The-Time-Machine_35',
 'Frankenstein_84',
 'Middlemarch_145',
 'A-Tale-of-Two-Cities_98',
 'The-Call-of-the-Wild_215',
 'Crime-and-Punishment_2554',
 'The-Strange-Case-of-Dr.-Jekyll-and-Mr.-Hyde_42',
 'Dracula_345',
 'Flatland--A-Romance-of-Many-Dimensions--Illustrated-_201',
 'Household-Stories-by-the-Brothers-Grimm_19068',
 'Heart-of-Darkness_219',
 'A-Journey-into-the-Interior-of-the-Earth_3748',
 'Jude-the-Obscure_153',
 'King-Solomon-s-Mines_2166',
 'Little-Women_514',
 'Madame-Bovary_2413',
 'The-Life-and-Adventures-of-Robinson-Crusoe_521',
 'The-Awakening-and-Selected-Short-Stories_160',
 'The-Jungle_140',
 'The-Jungle-Book_236',
 'Metamorphosis_5200',
 'The-Picture-of-Dorian-Gray_174',
 'The-Red-Badge-of-Courage_73',
 'The-Scarlet-Letter_33',
 'The-War-of-the-Worlds_36',
 'The-Wonderful-Wizard-of-Oz_55',
 'This-Side-of-Paradise_805',
 'Anna-Karenina_1399',
 'Gulliver-s-Travels_829',
 'Les-Mis-rables_135',
 'Swann-s-Way_7178',
 'The-Count-of-Monte-Cristo_1184',
 'The-Hunchback-of-Notre-Dame_6539',
 'The-Three-Musketeers_1257',
 'Through-the-Looking-Glass_12',
 'Twenty-Thousand-Leagues-under-the-Sea_164',
 'War-and-Peace_2600',
 'Winesburg-Ohio--A-Group-of-Tales-of-Ohio-Small-Town-Life_416',
 'My-Antonia_242',
 'Divine-Comedy-Longfellow-s-Translation-Hell_1001',
 'The-Works-of-Edgar-Allan-Poe-The-Raven-EditionTable-Of-Contents-And-Index-Of-The-Five-Volumes_25525']

In [6]:

    

repo = all_repos[2]
repo


    

    
Out[6]:





'Dubliners_2814'

In [ ]:

    

list(apply_to_repos(repo_version,kwargs={'version_type':'patch'},repos=all_repos))


    

In [9]:

    

def new_travis_template(repo, template, write_template=False):
    """
    compute (and optionally write) .travis.yml based on the template and current metadata.yaml 
    """
    template_written = False
    
    sh.cd(os.path.join(GITENBERG_DIR, repo))

    metadata_path = os.path.join(GITENBERG_DIR, repo, "metadata.yaml")
    travis_path = os.path.join(GITENBERG_DIR, repo, ".travis.yml")
    travis_api_key_path = os.path.join(GITENBERG_DIR, repo, ".travis.deploy.api_key.txt") 
    
    md = metadata.pandata.Pandata(metadata_path)
    epub_title = slugify(md.metadata.get("title"))
    encrypted_key = open(travis_api_key_path).read().strip()
    repo_name = md.metadata.get("_repo")
    
    template_vars =  {
        'epub_title': epub_title,
        'encrypted_key': encrypted_key,
        'repo_name': repo_name
    }
    
    template_result = template.render(**template_vars)
    
    if write_template:
        with open(travis_path, "w") as f:
            f.write(template_result)
        template_written = True
    
    return (template_result, template_written)


    

In [ ]:

    

from itertools import izip

template = template = travis_template()

results = list(izip(all_repos, apply_to_repos(new_travis_template,
                                        kwargs={'template':template},
                                        repos=all_repos)))
[result for result in results if isinstance(result[1], Exception) ]


    

In [ ]:

    

import os
import yaml
import pdb

def commit_travis_api_key_and_update_travis(repo, template, write_updates=False):
    """
    create .travis.deploy.api_key.txt and update .travis.yml; do git commit
    """
    sh.cd(os.path.join(GITENBERG_DIR, repo))

    metadata_path = os.path.join(GITENBERG_DIR, repo, "metadata.yaml")
    travis_path = os.path.join(GITENBERG_DIR, repo, ".travis.yml")
    travis_api_key_path = os.path.join(GITENBERG_DIR, repo, ".travis.deploy.api_key.txt") 
    
    # git add .travis.deploy.api_key.txt
    
    if write_updates:
        sh.git.add(travis_api_key_path)
    
    # read the current metadata file and replace current_ver with next_ver

    (v0, v1, v_updated) = repo_version(repo, version_type='patch', write_version=write_updates)
    if v_updated:
        sh.git.add(metadata_path)
        
    # write new .travis.yml
    (new_template, template_written) = new_travis_template(repo, template, write_template=write_updates)
    if template_written:
        sh.git.add(travis_path)
    
    if write_updates:
        sh.git.commit("-m", "add .travis.deploy.api_key.txt; updated .travis.yml")
    
    # add tag
    if v_updated:
        sh.git.tag(v1)
        sh.git.push("origin", "master", "--tags")

        return True

    else:
        return False


    

In [ ]:

    

problem_repos = ('The-Picture-of-Dorian-Gray_174',
                 'The-Hunchback-of-Notre-Dame_6539', 
                 'Divine-Comedy-Longfellow-s-Translation-Hell_1001',
                 'The-Works-of-Edgar-Allan-Poe-The-Raven-EditionTable-Of-Contents-And-Index-Of-The-Five-Volumes_25525'
                )


    

In [ ]:

    

repos = all_repos[36:][0:]
repos = [repo for repo in repos if repo not in problem_repos]
repos


    

In [ ]:

    

template = travis_template()

# I wish there would be a way to figure out variables in a template from jinja2...but I don't see a way.


    

In [ ]:

    

results = list(apply_to_repos(commit_travis_api_key_and_update_travis, 
               kwargs={'template':template,
                        'write_updates':True},
               repos=repos))
results


    

In [ ]:

    

import requests

def url_status(url):
    r = requests.get(url, allow_redirects=True, stream=True)
    return r.status_code

def repo_epub_status(repo):
    return url_status(latest_epub(repo))


    

In [ ]:

    

list(izip(repos, apply_to_repos(repo_epub_status, 
               repos=repos)))


    

In [ ]:

    

results = list(izip(all_repos, apply_to_repos(repo_epub_status, 
               repos=all_repos)))


    

In [ ]:

    

results


    

In [ ]:

    

ok_repos = [result[0] for result in results if result[1] == 200 ]
not_ok_repos = [result[0] for result in results if result[1] <> 200 ]
len(ok_repos), len(not_ok_repos)


    

In [ ]:

    

for (i, repo) in enumerate(ok_repos):
    print (i+1, "\t", repo, "\t", latest_epub(repo))


    

In [ ]:

    

not_ok_repos


    

In [12]:

    

from second_folio import TRAVIS_TEMPLATE_URL

repo = "Divine-Comedy-Longfellow-s-Translation-Hell_1001"


    

In [13]:

    

title = "Divine Comedy, Longfellow's Translation, Hell"
slugify(title)


    

    
Out[13]:





u'Divine-Comedy-Longfellows-Translation-Hell'

In [ ]: