In [ ]:

    

import requests
import json

# r = requests.get('https://ctftime.org/api/v1/teams/')

# with open('teams.json', 'w') as outfile:
#     json.dump(r.json(), outfile)

result = {}
with open('teams.json', 'r') as inpfile:
    result = json.loads(inpfile.read())

out = []
with open('teams_ru.json', 'w') as outfile:
    for team in result:
        if team['country'] == "RU": # and team['academic']:
            out.append(team)
#     print(len(out))
    json.dump(out, outfile, indent=4, sort_keys=True)


    

In [3]:

    

import re
import os
import urllib.request
import json
import requests


print("Starting....")
result = {}
with open('teams_ru.json', 'r') as inpfile:
    result = json.loads(inpfile.read())
    
pattern = re.compile('\<img src\=\"(.+)?" width="(.+)?" height="(.+)?">')
teams = []

print("Total teams: {}".format(len(result)))
for team in result:
    out = {}
    years = []
    r1 = requests.get('https://ctftime.org/team/{}'.format(str(team['id'])))
    r2 = requests.get('https://ctftime.org/api/v1/teams/{}/'.format(str(team['id'])))
    
    kek = pattern.findall(r1.text)
    api_team = r2.json()

    if len(api_team['rating']) == 0:
        continue
    for i, val in enumerate(api_team['rating']):
        for i2, val2 in enumerate(val):
            years.append(int(val2))
    if len(kek) == 1:
        out['ctftime_logo'] = "https://ctftime.org/{}".format(kek[0][0])
    else: 
        out['ctftime_logo'] = "https://ctftime.org/static/images/nologo.png"
    out['logo'] = 'teams/{}.png'.format(str(team['id']))
    out['name'] = team['name']
    out['ctftime_id'] = team['id']
    out['city'] = ''
    out['info'] = ''
    out['site'] = ''
    out['contacts'] = ''
    out['titles'] = ''
    out['coures'] = False
    out['academic'] = team['academic']
    out['disband'] = max(years)
    out['created'] = min(years)
    teams.append(out)
    
    if 'nologo.png' not in out['ctftime_logo'] and not os.path.isfile(out['logo']):
        urllib.request.urlretrieve(out['ctftime_logo'], out['logo'])
        
    
print("Team {} added".format(team['name']))
#     print(out)
with open('teams_ru_parsed.json', 'w') as outfile:
#     simplejson.dumps(simplejson.loads(output), indent=4, sort_keys=True)
    json.dump(teams, outfile, indent=4, sort_keys=True)
    Print("Finished!")


    

In [21]:

    

import json
teams = {}

with open('teams_ru_parsed.json', 'r') as inpfile:
    teams = json.loads(inpfile.read())


# for team in teams:
#     print(team)

with open('teams_ru_parsed.sql', 'w') as sqlfile:
    for team in teams:
        if team['disband'] >= 2015:
            team['academic'] = int(team['academic'] == True)
#             print(team['academic'])
            sqlfile.write("INSERT INTO teams (name, type, created, disband, logo, ctftime_logo, ctftime_id) VALUES ('{}', {}, {}, {}, '{}', '{}', {});\n"
                     .format(team['name'], team['academic'], team['created'], team['disband'], team['logo'], team['ctftime_logo'], team['ctftime_id']))

#          sqlfile.write("INSERT INTO teams(name, desc, type, created, disband, logo, ctftime_logo, ctftime_id, site, contancts, status, titles) VALUES ({}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {});\n"
#                       .format(out['name'], out['city']))


    

In [ ]: