We use an XML export of the various tables in the FileMaker Inkind database.
The XML will be read, field definitions will be extracted from it, the data will be read. We do the following:
In [31]:
import os,sys,re,collections,json
from os.path import splitext, basename
from functools import reduce
from glob import glob
from lxml import etree
from datetime import datetime
from pymongo import MongoClient
from bson.objectid import ObjectId
In [32]:
HOME_DIR = os.path.expanduser('~').replace('\\', '/')
BASE_DIR = '{}/Documents/DANS/projects/has/dacs'.format(HOME_DIR)
FM_DIR = '{}/fm'.format(BASE_DIR)
FMNS = '{http://www.filemaker.com/fmpxmlresult}'
CONFIG_DIR = '.'
In [ ]:
with open('{}/config.yaml')
In [ ]:
CONFIG = yaml.load('''
mainTables:
- contrib
- country
''')
In [33]:
mainTables = ('contrib', 'country')
SKIP_FIELDS = dict(
contrib=set('''
dateandtime_ciozero
ikid
ikid_base
find_country_id
find_type
gnewpassword
gnewpassword2
goldpassword
help_description
help_text
message
message_allert
teller
total_costs_total
whois
'''.strip().split()),
country=set('''
'''.strip().split()),
)
In [34]:
MERGE_FIELDS = dict(
contrib=dict(
academic_entity_url=['academic_entity_url_2'],
contribution_url=['contribution_url_2'],
contact_person_mail=['contact_person_mail_2'],
type_of_inkind=['other_type_of_inkind'],
vcc11_name=[
'vcc12_name',
'vcc21_name',
'vcc22_name',
'vcc31_name',
'vcc32_name',
'vcc41_name',
'vcc42_name',
],
vcc_head_decision_vcc11=[
'vcc_head_decision_vcc12',
'vcc_head_decision_vcc21',
'vcc_head_decision_vcc22',
'vcc_head_decision_vcc31',
'vcc_head_decision_vcc32',
'vcc_head_decision_vcc41',
'vcc_head_decision_vcc42',
],
),
country=dict(),
)
In [35]:
MAP_FIELDS = dict(
contrib=dict(
approved='approved',
academic_entity_url='urlAcademic',
contribution_url='urlContribution',
contact_person_mail='contactPersonEmail',
contact_person_name='contactPersonName',
costs_description='costDescription',
costs_total='costTotal',
country='country',
creation_date_time='dateCreated',
creator='creator',
dateandtime_approval='dateApproved',
dateandtime_cioapproval='dateApprovedCIO',
description_of_contribution='description',
disciplines_associated='discipline',
last_modifier='modifiedBy',
modification_date_time='dateModified',
other_keywords='keyword',
submit='submitted',
tadirah_research_activities='tadirahActivity',
tadirah_research_objects='tadirahObject',
tadirah_research_techniques='tadirahTechnique',
title='title',
total_costs_total='costTotalTotal',
type_of_inkind='typeContribution',
vcc='vcc',
vcc11_name='reviewerName',
vcc_head_decision='vccDecision',
vcc_head_decision_vcc11='reviewerDecision',
vcchead_approval='vccApproval',
vcchead_disapproval='vccDisApproval',
year='year',
),
country=dict(
countrycode='iso',
countryname='name',
member_dariah='isMember',
),
)
In [36]:
generic = re.compile('[ \t]*[\n+][ \t\n]*') # split on newlines (with surrounding white space)
genericComma = re.compile('[ \t]*[\n+,;][ \t\n]*') # split on newlines or commas (with surrounding white space)
SPLIT_FIELDS=dict(
contrib=dict(
discipline=generic,
keyword=genericComma,
typeContribution=generic,
tadirahActivity=generic,
tadirahObject=generic,
tadirahTechnique=generic,
vcc=generic,
),
country=dict(),
)
In [37]:
STRIP_NUM = re.compile('^[0-9]\s*\.?\s+')
def stripNum(v): return STRIP_NUM.sub('', v)
HACK_FIELDS=dict(
contrib=dict(
tadirahActivity=stripNum,
),
country=dict(),
)
In [38]:
DECOMPOSE_FIELDS=dict(
contrib=dict(
typeContribution='typeContributionOther',
),
country=dict(),
)
In [39]:
FIELD_TYPE = dict(
contrib=dict(
costTotal='valuta',
dateCreated='datetime',
dateModified='datetime',
dateApproved='datetime',
dateApprovedCIO='datetime',
contactPersonEmail='email',
submitted='bool',
approved='bool',
reviewerDecision='bool',
vccApproval='bool',
vccDecision='bool',
vccDisApproval='bool',
),
country=dict(
isMember='bool',
),
)
In [40]:
DEFAULT_VALUES=dict(
contrib=dict(
dateCreated=datetime(2000,1,1,0,0,0),
creator="admin",
type_of_inkind="General",
),
country=dict(),
)
In [41]:
MOVE_FIELDS=dict(
contrib=dict(
assessment=set('''
approved
dateApproved
dateApprovedCIO
submitted
reviewerName
reviewerDecision
vccDecision
vccApproval
vccDisApproval
'''.strip().split()),
),
country=dict(),
)
In [42]:
MAKE_VALUE_LISTS = dict(
contrib=set('''
keyword
year
'''.strip().split()),
)
VALUE_LISTS = dict(
contrib=set('''
discipline
keyword
tadirahActivity
tadirahObject
tadirahTechnique
typeContribution
typeContributionOther:typeContribution
vcc
year
'''.strip().split()),
)
MOVE_MISSING = dict(
contrib='description',
)
In [43]:
# Source field types, including types assigned by type overriding (see FIELD_TYPE_OVERRIDE above).
# These will be translated into appropriate SQL field types
TYPES = {'bool', 'number', 'decimal', 'text', 'valuta', 'email', 'date', 'datetime'}
# dates are already in ISO (date2_pattern).
# If we encounter other dates, we could use date_pattern instead)
# datetimes are not in iso, they will be transformed to iso.
DECIMAL_PATTERN = re.compile(
r'^-?[0-9]+\.?[0-9]*'
)
DATE_PATTERN = re.compile(
r'^\s*([0-9]{2})/([0-9]{2})/([0-9]{4})$'
)
DATE2_PATTERN = re.compile(
r'^\s*([0-9]{4})-([0-9]{2})-([0-9]{2})$'
)
DATETIME_PATTERN = re.compile(
r'^\s*([0-9]{2})/([0-9]{2})/([0-9]{4})\s+([0-9]{2}):([0-9]{2})(?::([0-9]{2}))?$'
)
# meaningless values will be translated into None
NULL_VALUES = {
'http://',
'https://',
'@',
}
BOOL_VALUES = {
True: {'Yes', 'YES', 'yes', 1, '1', True},
False: {'No', 'NO', 'no', 0, '0', 'NULL', False},
}
In [44]:
def date_repl(match):
[d,m,y] = list(match.groups())
return '{}-{}-{}'.format(y,m,d)
def date2_repl(match):
[y,m,d] = list(match.groups())
return '{}-{}-{}'.format(y,m,d)
def datetime_repl(match):
[d,m,y,hr,mn,sc] = list(match.groups())
return '{}-{}-{}T{}:{}:{}'.format(y,m,d,hr,mn,sc or '00')
def dt(v_raw, i, t, fname):
if not DATE2_PATTERN.match(v_raw):
warning(
'table `{}` field `{}` record {}: not a valid date: "{}"'.format(
t, fname, i, v_raw
))
return v_raw
return datetime(*map(int, re.split('[:T-]', DATE2_PATTERN.sub(date2_repl, v_raw))))
def dtm(v_raw, i, t, fname):
if not DATETIME_PATTERN.match(v_raw):
warning(
'table `{}` field `{}` record {}: not a valid date time: "{}"'.format(
t, fname, i, v_raw
))
return v_raw
return datetime(*map(int, re.split('[:T-]', DATETIME_PATTERN.sub(datetime_repl, v_raw))))
In [45]:
def bools(v_raw, i, t, fname):
if v_raw in BOOL_VALUES[True]: return True
if v_raw in BOOL_VALUES[False]: return False
warning(
'table `{}` field `{}` record {}: not a boolean value: "{}"'.format(
t, fname, i, v_raw
))
return v_raw
def num(v_raw, i, t, fname):
if type(v_raw) is int: return v_raw
if v_raw.isdigit(): return int(v_raw)
warning(
'table `{}` field `{}` record {}: not an integer: "{}"'.format(
t, fname, i, v_raw
))
return v_raw
def decimal(v_raw, i, t, fname):
if type(v_raw) is float: return v_raw
if v_raw.isdigit(): return float(v_raw)
if DECIMAL_PATTERN.match(v_raw): return float(v_raw)
warning(
'table `{}` field `{}` record {}: not an integer: "{}"'.format(
t, fname, i, v_raw
))
return v_raw
def email(v_raw, i, t, fname):
return v_raw.replace('mailto:', '', 1) if v_raw.startswith('mailto:') else v_raw
In [46]:
def money(v_raw, i, t, fname):
note = ',' in v_raw or '.' in v_raw
v = v_raw.strip().lower().replace(' ','').replace('€', '').replace('euro', '').replace('\u00a0', '')
for p in range(2,4): # interpret . or , as decimal point if less than 3 digits follow it
if len(v) >= p and v[-p] in '.,':
v_i = v[::-1]
if v_i[p-1] == ',': v_i = v_i.replace(',', 'D', 1)
elif v_i[p-1] == '.': v_i = v_i.replace('.', 'D', 1)
v = v_i[::-1]
v = v.replace('.','').replace(',','')
v = v.replace('D', '.')
if not v.replace('.','').isdigit():
if len(set(v) & set('0123456789')):
warning(
'table `{}` field `{}` record {}: not a decimal number: "{}" <= "{}"'.format(
t, fname, i, v, v_raw,
))
money_warnings.setdefault('{}:{}'.format(t, fname), {}).setdefault(v, set()).add(v_raw)
v = None
else:
v = None
money_notes.setdefault('{}:{}'.format(t, fname), {}).setdefault('NULL', set()).add(v_raw)
elif note:
money_notes.setdefault('{}:{}'.format(t, fname), {}).setdefault(v, set()).add(v_raw)
return None if v == None else float(v)
In [47]:
def sanitize(t, i, fname, value):
if fname == '_id': return value
(ftype, fmult) = allFields[t][fname]
newValue = []
for v_raw in value:
if v_raw == None or v_raw in NULL_VALUES: continue
elif ftype == 'text': v = v_raw
elif ftype == 'bool': v = bools(v_raw, i, t, fname)
elif ftype == 'number': v = num(v_raw, i, t, fname)
elif ftype == 'decimal': v = decimal(v_raw, i, t, fname)
elif ftype == 'email': v = email(v_raw, i, t, fname)
elif ftype == 'valuta': v = money(v_raw, i, t, fname)
elif ftype == 'date': v = dt(v_raw, i, t, fname)
elif ftype == 'datetime': v = dtm(v_raw, i, t, fname)
else: v = v_raw
if v != None and (fmult <= 1 or v != ''): newValue.append(v)
if len(newValue) == 0:
defValue = DEFAULT_VALUES.get(t, {}).get(fname, None)
if defValue != None:
newValue = [defValue]
return newValue
In [48]:
def info(x): sys.stdout.write('{}\n'.format(x))
def warning(x): sys.stderr.write('{}\n'.format(x))
def showFields():
for (mt, defs) in sorted(allFields.items()):
info(mt)
for (fname, fdef) in sorted(defs.items()):
info('{:>25}: {:<10} ({})'.format(fname, *fdef))
def showdata(rows):
for row in rows:
for f in sorted(row.items()):
info('{:>20} = {}'.format(*f))
info('o-o-o-o-o-o-o-o-o-o-o-o')
def showData():
for (mt, rows) in sorted(allData.items()):
info('o-o-o-o-o-o-o TABLE {} with {} rows o-o-o-o-o-o-o-o '.format(mt, len(rows)))
showdata(rows[0:2])
def showMoney():
for tf in sorted(money_notes):
for v in sorted(money_notes[tf]):
info('{} "{}" <= {}'.format(
tf, v,
' | '.join(money_notes[tf][v]),
))
In [49]:
def readFmFields():
for mt in mainTables:
infile = '{}/{}.xml'.format(FM_DIR, mt)
root = etree.parse(infile, parser).getroot()
fieldroots = [x for x in root.iter(FMNS+'METADATA')]
fieldroot = fieldroots[0]
fields = []
fieldDefs = {}
for x in fieldroot.iter(FMNS+'FIELD'):
fname = x.get('NAME').lower().replace(' ','_').replace(':', '_')
ftype = x.get('TYPE').lower()
fmult = int(x.get('MAXREPEAT'))
fields.append(fname)
fieldDefs[fname] = [ftype, fmult]
rawFields[mt] = fields
allFields[mt] = fieldDefs
for f in SKIP_FIELDS[mt]:
del allFields[mt][f]
for (f, mfs) in MERGE_FIELDS[mt].items():
allFields[mt][f][1] += 1
for mf in mfs:
del allFields[mt][mf]
allFields[mt] = dict((MAP_FIELDS[mt][f], v) for (f,v) in allFields[mt].items())
for f in SPLIT_FIELDS[mt]:
allFields[mt][f][1] += 1
for (f, fo) in DECOMPOSE_FIELDS[mt].items():
allFields[mt][fo] = allFields[mt][f]
allFields[mt][f] = [allFields[mt][f][0], 1]
for (f, t) in FIELD_TYPE[mt].items():
allFields[mt][f][0] = t
In [50]:
def readFmData():
for mt in mainTables:
infile = '{}/{}.xml'.format(FM_DIR, mt)
root = etree.parse(infile, parser).getroot()
dataroots = [x for x in root.iter(FMNS+'RESULTSET')]
dataroot = dataroots[0]
rows = []
rowsRaw = []
fields = rawFields[mt]
for (i, r) in enumerate(dataroot.iter(FMNS+'ROW')):
rowRaw = []
for c in r.iter(FMNS+'COL'):
data = [x.text.strip() for x in c.iter(FMNS+'DATA') if x.text != None]
rowRaw.append(data)
if len(rowRaw) != len(fields):
warning('row {}: fields encountered = {}, should be {}'.format(len(row), len(fields)))
rowsRaw.append(dict((f,v) for (f, v) in zip(fields, rowRaw)))
row = dict((f,v) for (f, v) in zip(fields, rowRaw) if f not in SKIP_FIELDS[mt])
for (f, mfs) in MERGE_FIELDS[mt].items():
for mf in mfs:
row[f].extend(row[mf])
del row[mf]
row = dict((MAP_FIELDS[mt][f], v) for (f,v) in row.items())
for (f, spl) in SPLIT_FIELDS[mt].items():
row[f] = reduce(lambda x,y: x+y, [spl.split(v) for v in row[f]], [])
for (f, hack) in HACK_FIELDS[mt].items():
row[f] = [hack(v) for v in row[f]]
for (f, fo) in DECOMPOSE_FIELDS[mt].items():
row[fo] = row[f][1:]
row[f] = [row[f][0]] if len(row[f]) else []
row['_id'] = ObjectId()
#info('\n'.join('{}={}'.format(*x) for x in sorted(row.items())))
for (f, v) in row.items(): row[f] = sanitize(mt, i, f, v)
rows.append(row)
allData[mt] = rows
rawData[mt] = rowsRaw
if money_warnings:
for tf in sorted(money_warnings):
for v in sorted(money_warnings[tf]):
warning('{} "{}" <= {}'.format(
tf, v,
' | '.join(money_warnings[tf][v]),
))
In [51]:
def moveFields():
for mt in mainTables:
for (omt, mfs) in MOVE_FIELDS[mt].items():
for mf in mfs:
allFields.setdefault(omt, dict())[mf] = allFields[mt][mf]
del allFields[mt][mf]
allFields.setdefault(omt, dict)['{}_id'.format(mt)] = ('id', 1)
for row in allData[mt]:
for (omt, mfs) in MOVE_FIELDS[mt].items():
orow = dict((mf, row[mf]) for mf in mfs)
orow['_id'] = ObjectId()
orow['{}_id'.format(mt)] = row['_id']
allData.setdefault(omt, []).append(orow)
for mf in mfs: del row[mf]
In [52]:
def readLists():
valueLists = dict()
for path in glob('{}/*.txt'.format(FM_DIR)):
tname = basename(splitext(path)[0])
data = []
with open(path) as fh:
for line in fh:
data.append(line.rstrip().split('\t'))
valueLists[tname] = data
for (vList, data) in valueLists.items():
if vList == 'countryExtra':
mapping = dict((x[0], x[1:]) for x in data)
else:
mapping = dict((i+1, x[0]) for (i, x) in enumerate(data))
valueDict[vList] = mapping
allFields[vList] = dict(
_id=('id', 1),
value=('string', 1),
)
for mt in allData:
fs = MAKE_VALUE_LISTS.get(mt, set())
for f in fs:
valSet = set()
for row in allData[mt]:
values = row.get(f, [])
if type(values) is not list:
values = [values]
valSet |= set(values)
valueDict[f] = dict((i+1, x) for (i, x) in enumerate(sorted(valSet)))
allFields[f] = dict(
_id=('id', 1),
value=('string', 1),
)
In [53]:
def countryTable():
extraInfo = valueDict['countryExtra']
idMapping = dict()
for row in allData['country']:
for f in row:
if type(row[f]) is list: row[f] = row[f][0]
iso = row['iso']
row['_id'] = ObjectId()
idMapping[iso] = row['_id']
(name, lat, long) = extraInfo[iso]
row['latitude'] = lat
row['longitude'] = long
for row in allData['contrib']:
newValue = []
for iso in row['country']:
newValue.append(dict(_id=idMapping[iso], iso=iso, value=extraInfo[iso][0]))
row['country'] = newValue
allFields['country']['_id'] = ('id', 1)
allFields['country']['iso'] = ('string', 1)
allFields['country']['latitude'] = ('float', 1)
allFields['country']['longitude'] = ('float', 1)
In [54]:
def userTable():
idMapping = dict()
existingUsers = []
testUsers = [
dict(eppn='suzan', email='suzan1@test.eu', mayLogin=True, authority='local',
firstName='Suzan', lastName='Karelse'),
dict(eppn='marie', email='suzan2@test.eu', mayLogin=True, authority='local',
firstName='Marie', lastName='Pieterse'),
dict(eppn='gertjan', email='gertjan@test.eu', mayLogin=False, authority='local',
firstName='Gert Jan', lastName='Klein-Holgerink'),
dict(eppn='lisa', email='lisa@test.eu', mayLogin=True, authority='local',
firstName='Lisa', lastName='de Leeuw'),
dict(eppn='dirk', email='dirk@test.eu', mayLogin=True, authority='local',
firstName='Dirk', lastName='Roorda'),
]
users = collections.defaultdict(set)
eppnSet = set()
for c in allData['contrib']:
crs = c.get('creator', []) + c.get('modifiedBy', [])
for cr in crs:
eppnSet.add(cr)
idMapping = dict((eppn, ObjectId()) for eppn in sorted(eppnSet))
for c in allData['contrib']:
c['creator'] = [dict(_id=idMapping[cr]) for cr in c['creator']]
if 'modifiedBy' not in c:
c['modifiedBy'] = []
else:
c['modifiedBy'] = [dict(_id=idMapping[cr]) for cr in c['modifiedBy']]
users = dict((i, eppn) for (eppn, i) in idMapping.items())
for (i, eppn) in sorted(users.items()):
existingUsers.append(dict(_id=i, eppn=eppn, mayLogin=False, authority='legacy'))
for u in testUsers:
u['_id'] = ObjectId()
idMapping[u['eppn']] = u['_id']
existingUsers.append(u)
inGroups = [
dict(eppn='DirkRoorda@dariah.eu', authority='DARIAH', group='system'),
dict(eppn='LisaDeLeeuw@dariah.eu', authority='DARIAH', group='office'),
dict(eppn='suzan', authority='local', group='auth'),
dict(eppn='marie', authority='local', group='auth'),
dict(eppn='gertjan', authority='local', group='auth'),
dict(eppn='lisa', authority='local', group='office'),
dict(eppn='dirk', authority='local', group='system'),
]
inGroups = [dict(tuple(ig.items())+(('_id', ObjectId()),)) for ig in inGroups]
allData['user'] = existingUsers
allData['group'] = inGroups
allFields['user'] = dict(
_id=('id', 1),
eppn=('string', 1),
email=('email', 1),
mayLogin=('bool', 1),
authority=('string', 1),
firstName=('string', 1),
lastName=('string', 1),
)
allFields['group'] = dict(
_id=('id', 1),
eppn=('string', 1),
authority=('string', 1),
group=('string', 1),
)
uidMapping.update(idMapping)
In [55]:
def relTables():
def norm(x): return x.strip().lower()
relIndex = dict()
for mt in sorted(VALUE_LISTS):
rows = allData[mt]
for f in sorted(VALUE_LISTS[mt]):
comps = f.split(':')
if len(comps) == 2:
(f, fAs) = comps
else:
fAs = f
relInfo = valueDict[fAs]
if not fAs in relIndex:
idMapping = dict((i, ObjectId()) for i in relInfo)
allData[fAs] = [dict(_id=idMapping[i], value=v) for (i, v) in relInfo.items()]
relIndex[fAs] = dict((norm(v), (idMapping[i], v)) for (i, v) in relInfo.items())
for row in rows:
newValue = []
for v in row[f]:
rnv = norm(v)
(i, nv) = relIndex[fAs].get(rnv, ("-1", None))
if nv == None:
target = MOVE_MISSING[mt]
if target not in row: row[target] = ['']
row[target][0] += '\nMOVED FROM {}: {}'.format(f, v)
else: newValue.append(dict(_id=i, value=nv))
row[f] = newValue
In [56]:
def testTweaks():
mt = 'contrib'
myContribs = {'3DHOP', 'AAI'}
my = uidMapping['dirk']
for row in allData[mt]:
title = row.get('title', [None])
if len(title) == 0: title = [None]
if title[0] in myContribs:
row['creator'] = [dict(_id=my)]
In [57]:
def importMongo():
client = MongoClient()
client.drop_database('dariah')
db = client.dariah
for (mt, rows) in allData.items():
info(mt)
db[mt].insert_many(rows)
In [58]:
money_warnings = {}
money_notes = {}
valueDict = dict()
rawFields = dict()
allFields = dict()
rawData = dict()
allData = dict()
uidMapping = dict()
parser = etree.XMLParser(remove_blank_text=True, ns_clean=True)
readFmFields()
readFmData()
readLists()
moveFields()
countryTable()
userTable()
relTables()
testTweaks()
importMongo()
#showData()
#showMoney()
To import the bson dump in another mongodb installation, use the commandline to dump the dariah database here
mongodump -d dariah -o dariah
and to import it elsewhere.
mongorestore --drop -d dariah dariah
In [33]:
valueDict.keys()
Out[33]:
In [54]:
valueDict['keywords']
Out[54]:
In [106]:
import xlsxwriter
EXPORT_DIR = os.path.expanduser('~/Downloads')
EXPORT_ORIG = '{}/contribFromFileMaker.xlsx'.format(EXPORT_DIR)
EXPORT_MONGO = '{}/contribInMongoDB.xlsx'.format(EXPORT_DIR)
In [107]:
workbook = xlsxwriter.Workbook(EXPORT_ORIG, {'strings_to_urls': False})
for mt in rawData:
worksheet = workbook.add_worksheet(mt)
for (f, field) in enumerate(rawFields[mt]):
worksheet.write(0, f, field)
for (r, row) in enumerate(rawData[mt]):
for (f, field) in enumerate(rawFields[mt]):
val = row[field]
val = [] if val == None else val if type(val) is list else [val]
val = '|'.join(val)
worksheet.write(r+1, f, val)
workbook.close()
In [108]:
workbook = xlsxwriter.Workbook(EXPORT_MONGO, {'strings_to_urls': False})
for mt in allData:
worksheet = workbook.add_worksheet(mt)
fields = sorted(allFields[mt])
for (f, field) in enumerate(fields):
worksheet.write(0, f, field)
for (r, row) in enumerate(allData[mt]):
for (f, field) in enumerate(fields):
fmt = None
val = row.get(field, [])
(ftype, fmult) = allFields[mt][field]
val = [] if val == None else [val] if type(val) is not list else val
exportVal = []
for v in val:
if type(v) is dict:
exportVal.append(','.join(str(vv) for vv in v.values()))
elif ftype == 'date' or ftype == 'datetime':
exportVal.append(v if type(v) is str else v.isoformat())
else:
exportVal.append(str(v))
worksheet.write(r+1, f, ' | '.join(exportVal))
workbook.close()
In [109]:
showFields()
In [30]:
client = MongoClient()
dbm = client.dariah
for d in dbm.contrib.find({'title': '3DHOP'}).limit(2):
print('=' * 50)
for f in sorted(d):
print('{}={}'.format(f, d[f]))
Here is a query to get all 'type_of_inkind' values for contributions.
In [32]:
for c in dbm.contrib.distinct('typeContribution', {}):
print(c)
Here are the users:
In [33]:
for c in dbm.users.find({}):
print(c)
Here are the countries:
In [34]:
for c in dbm.country.find({'isMember': True}):
print(c)
In [35]:
for c in dbm.contrib.distinct('country', {}):
print(c)
Let us get related data: the type_of_inkind of all contributions. For each contribution we need only the ids of the related type_of_inkind values.
In [39]:
for d in dbm.contrib.find({}, {'typeContribution': True}).limit(10):
print(d)
In [40]:
for d in dbm.contrib.find({}, {'country': True}).limit(10):
print(d)
In [29]:
x = dict(_id=5, value='66')
y = dict(_id=5, value='66')
x == y
Out[29]:
In [ ]: