notebook.community

Edit and run



In [24]:

    
import requests
import os
import errno
def mkdir_p(path):
    """Python version mkdir -p
    Parameters
    ----------
    path : str
    """
    if path:
        try:
            os.makedirs(path)
        except OSError as exc:  # Python >2.5
            if exc.errno == errno.EEXIST and os.path.isdir(path):
                pass
            else:
                raise



In [30]:

    
BASE_URL = 'ftp.ensemblgenomes.org'#/pub/release-46/fungi/fasta/'
WRITE_DATA_DIR = './tmp/fungi_genomes_pep'



In [31]:

    
import ftplib

with ftplib.FTP(BASE_URL) as ftp:
    print(ftp.getwelcome())
    ftp.login()           
    print(ftp.retrlines('LIST'))









    



220-
220- ftp.ensemblgenomes FTP server
220 
drwxr-sr-x   64 ftp      ftp          8192 Jan 16 10:48 pub
-rw-r--r--    1 ftp      ftp            26 Dec 05  2014 robots.txt
226 Directory send OK.



In [32]:

    
with ftplib.FTP(BASE_URL) as ftp:
    print(ftp.getwelcome())
    ftp.login()           
    print(ftp.retrlines('LIST'))
    ftp.cwd('/pub/release-46/fungi/fasta/')               # change into "debian" directory
    ftp.retrlines('LIST')









    



220-
220- ftp.ensemblgenomes FTP server
220 
drwxr-sr-x   64 ftp      ftp          8192 Jan 16 10:48 pub
-rw-r--r--    1 ftp      ftp            26 Dec 05  2014 robots.txt
226 Directory send OK.
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:07 ashbya_gossypii
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 21:43 aspergillus_clavatus
drwxrwsr-x    7 ftp      ftp          4096 Nov 19 03:57 aspergillus_flavus
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 04:17 aspergillus_fumigatus
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 06:15 aspergillus_fumigatusa1163
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 01:22 aspergillus_nidulans
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:13 aspergillus_niger
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:49 aspergillus_oryzae
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:21 aspergillus_terreus
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:15 beauveria_bassiana
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:58 blumeria_graminis
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 22:39 botrytis_cinerea
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 01:28 colletotrichum_gloeosporioides
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:29 colletotrichum_graminicola
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 04:18 colletotrichum_higginsianum
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 00:29 colletotrichum_orbiculare
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 05:40 cryptococcus_neoformans
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 20:34 dothistroma_septosporum
drwxrwsr-x  198 ftp      ftp         36864 Nov 18 14:43 fungi_ascomycota1_collection
drwxrwsr-x  158 ftp      ftp         28672 Nov 18 14:43 fungi_ascomycota2_collection
drwxrwsr-x  240 ftp      ftp         45056 Nov 18 14:40 fungi_ascomycota3_collection
drwxrwsr-x   91 ftp      ftp         16384 Nov 18 14:39 fungi_ascomycota4_collection
drwxrwsr-x  207 ftp      ftp         40960 Nov 01 17:20 fungi_basidiomycota1_collection
drwxrwsr-x    4 ftp      ftp          4096 Nov 18 14:51 fungi_blastocladiomycota1_collection
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 14:32 fungi_chytridiomycota1_collection
drwxrwsr-x    4 ftp      ftp          4096 Nov 18 14:45 fungi_entomophthoromycota1_collection
drwxrwsr-x   32 ftp      ftp          8192 Nov 18 14:39 fungi_microsporidia1_collection
drwxrwsr-x   30 ftp      ftp          8192 Nov 18 14:51 fungi_mucoromycota1_collection
drwxrwsr-x    6 ftp      ftp          4096 Nov 18 14:34 fungi_neocallimastigomycota1_collection
drwxrwsr-x    4 ftp      ftp          4096 Nov 18 14:45 fungi_rozellomycota1_collection
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 22:03 fusarium_culmorum
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 15:16 fusarium_fujikuroi
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 17:37 fusarium_graminearum
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 19:52 fusarium_oxysporum
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 02:53 fusarium_pseudograminearum
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 21:45 fusarium_solani
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 14:32 fusarium_verticillioides
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:42 gaeumannomyces_graminis
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 19:50 komagataella_pastoris
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 05:18 leptosphaeria_maculans
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 03:50 magnaporthe_oryzae
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 23:36 magnaporthe_poae
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 04:34 melampsora_laricipopulina
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 21:38 microbotryum_violaceum
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 19:03 neosartorya_fischeri
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 21:52 neurospora_crassa
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:35 phaeosphaeria_nodorum
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 05:52 puccinia_graminis
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 05:52 puccinia_graminisug99
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 00:09 puccinia_striiformis
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 14:35 puccinia_triticina
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 21:59 pyrenophora_teres
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 17:15 pyrenophora_triticirepentis
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 22:27 saccharomyces_cerevisiae
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 06:26 schizosaccharomyces_cryophilus
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 19:22 schizosaccharomyces_japonicus
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 21:30 schizosaccharomyces_octosporus
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 19:16 schizosaccharomyces_pombe
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 17:11 sclerotinia_sclerotiorum
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 20:37 sporisorium_reilianum
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 23:55 trichoderma_reesei
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 05:42 trichoderma_virens
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 22:22 tuber_melanosporum
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 06:03 ustilago_maydis
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 06:26 verticillium_dahliae
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 21:34 verticillium_dahliaejr2
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 21:54 yarrowia_lipolytica
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 20:16 zymoseptoria_tritici



In [ ]:

    
with ftplib.FTP(BASE_URL) as ftp:
    #print(ftp.getwelcome())
    ftp.login()           
    #print(ftp.retrlines('LIST'))
    ftp.cwd('/pub/release-46/fungi/fasta/')               # change into "debian" directory
    files = ftp.retrlines('LIST')
    #print(files)
    genomes = ftp.nlst()
    for genome in genomes:
        ftp.cwd("/pub/release-46/fungi/fasta/{}".format(genome))
        genome_directories = ftp.nlst()
        if 'pep' in genome_directories:
            ftp.cwd("/pub/release-46/fungi/fasta/{}/pep".format(genome))
            for ftp_file in ftp.nlst():
                dir_to_write = "{}/{}".format(WRITE_DATA_DIR, genome)
                file_to_write = os.path.join(dir_to_write, ftp_file)
                write_mode = 'w'
                if '.gz' in file_to_write:
                    write_mode = 'wb'
                mkdir_p(dir_to_write)
                print(ftp_file, write_mode)
                
                with open(file_to_write, write_mode) as fp:
                    if write_mode == "wb":
                        res = ftp.retrbinary('RETR ' + ftp_file, fp.write)                        
                    else:
                        res = ftp.retrlines('RETR ' + ftp_file, fp.write)
                    #ftp.retrbinary("RETR {}".format(vcf), open(vcf, 'wb').write)
                    if not res.startswith('226 Transfer complete'):
                        print('Download failed')
                        if os.path.isfile(file_to_write):
                            os.remove(file_to_write)









    



drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:07 ashbya_gossypii
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 21:43 aspergillus_clavatus
drwxrwsr-x    7 ftp      ftp          4096 Nov 19 03:57 aspergillus_flavus
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 04:17 aspergillus_fumigatus
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 06:15 aspergillus_fumigatusa1163
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 01:22 aspergillus_nidulans
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:13 aspergillus_niger
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:49 aspergillus_oryzae
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:21 aspergillus_terreus
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:15 beauveria_bassiana
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:58 blumeria_graminis
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 22:39 botrytis_cinerea
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 01:28 colletotrichum_gloeosporioides
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:29 colletotrichum_graminicola
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 04:18 colletotrichum_higginsianum
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 00:29 colletotrichum_orbiculare
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 05:40 cryptococcus_neoformans
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 20:34 dothistroma_septosporum
drwxrwsr-x  198 ftp      ftp         36864 Nov 18 14:43 fungi_ascomycota1_collection
drwxrwsr-x  158 ftp      ftp         28672 Nov 18 14:43 fungi_ascomycota2_collection
drwxrwsr-x  240 ftp      ftp         45056 Nov 18 14:40 fungi_ascomycota3_collection
drwxrwsr-x   91 ftp      ftp         16384 Nov 18 14:39 fungi_ascomycota4_collection
drwxrwsr-x  207 ftp      ftp         40960 Nov 01 17:20 fungi_basidiomycota1_collection
drwxrwsr-x    4 ftp      ftp          4096 Nov 18 14:51 fungi_blastocladiomycota1_collection
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 14:32 fungi_chytridiomycota1_collection
drwxrwsr-x    4 ftp      ftp          4096 Nov 18 14:45 fungi_entomophthoromycota1_collection
drwxrwsr-x   32 ftp      ftp          8192 Nov 18 14:39 fungi_microsporidia1_collection
drwxrwsr-x   30 ftp      ftp          8192 Nov 18 14:51 fungi_mucoromycota1_collection
drwxrwsr-x    6 ftp      ftp          4096 Nov 18 14:34 fungi_neocallimastigomycota1_collection
drwxrwsr-x    4 ftp      ftp          4096 Nov 18 14:45 fungi_rozellomycota1_collection
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 22:03 fusarium_culmorum
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 15:16 fusarium_fujikuroi
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 17:37 fusarium_graminearum
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 19:52 fusarium_oxysporum
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 02:53 fusarium_pseudograminearum
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 21:45 fusarium_solani
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 14:32 fusarium_verticillioides
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:42 gaeumannomyces_graminis
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 19:50 komagataella_pastoris
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 05:18 leptosphaeria_maculans
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 03:50 magnaporthe_oryzae
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 23:36 magnaporthe_poae
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 04:34 melampsora_laricipopulina
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 21:38 microbotryum_violaceum
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 19:03 neosartorya_fischeri
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 21:52 neurospora_crassa
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 16:35 phaeosphaeria_nodorum
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 05:52 puccinia_graminis
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 05:52 puccinia_graminisug99
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 00:09 puccinia_striiformis
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 14:35 puccinia_triticina
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 21:59 pyrenophora_teres
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 17:15 pyrenophora_triticirepentis
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 22:27 saccharomyces_cerevisiae
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 06:26 schizosaccharomyces_cryophilus
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 19:22 schizosaccharomyces_japonicus
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 21:30 schizosaccharomyces_octosporus
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 19:16 schizosaccharomyces_pombe
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 17:11 sclerotinia_sclerotiorum
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 20:37 sporisorium_reilianum
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 23:55 trichoderma_reesei
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 05:42 trichoderma_virens
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 22:22 tuber_melanosporum
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 06:03 ustilago_maydis
drwxrwsr-x    8 ftp      ftp          4096 Nov 19 06:26 verticillium_dahliae
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 21:34 verticillium_dahliaejr2
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 21:54 yarrowia_lipolytica
drwxrwsr-x    8 ftp      ftp          4096 Nov 18 20:16 zymoseptoria_tritici
Ashbya_gossypii.ASM9102v1.pep.all.fa.gz wb
CHECKSUMS w
README w
Aspergillus_clavatus.ASM271v1.pep.all.fa.gz wb
CHECKSUMS w
README w
Aspergillus_flavus.JCVI-afl1-v2.0.pep.all.fa.gz wb
CHECKSUMS w
README w
Aspergillus_fumigatus.ASM265v1.pep.all.fa.gz wb
CHECKSUMS w
README w
Aspergillus_fumigatusa1163.ASM15014v1.pep.all.fa.gz wb
CHECKSUMS w
README w