In [6]:
import py_entitymatching as em
import os
import pandas as pd

# specify filepaths for tables A and B. 
path_A = 'newTableA.csv'
path_B = 'tableB.csv'
# read table A; table A has 'ID' as the key attribute
A = em.read_csv_metadata(path_A, key='id')
# read table B; table B has 'ID' as the key attribute
B = em.read_csv_metadata(path_B, key='id')


Metadata file is not present in the given path; proceeding to read the csv file.
Metadata file is not present in the given path; proceeding to read the csv file.

Filling in Missing Values


In [7]:
# Impute missing values

# Manually set metadata properties, as current py_entitymatching.impute_table()
# requires 'fk_ltable', 'fk_rtable', 'ltable', 'rtable' properties
em.set_property(A, 'fk_ltable', 'id')
em.set_property(A, 'fk_rtable', 'id')
em.set_property(A, 'ltable', A)
em.set_property(A, 'rtable', A)

A_all_attrs = list(A.columns.values)
A_impute_attrs = ['year','min_num_players','max_num_players','min_gameplay_time','max_gameplay_time','min_age']
A_exclude_attrs = list(set(A_all_attrs) - set(A_impute_attrs))
A1 = em.impute_table(A, exclude_attrs=A_exclude_attrs, missing_val='NaN', strategy='most_frequent', axis=0, val_all_nans=0, verbose=True)

# Compare number of missing values to check the results
print(sum(A['min_num_players'].isnull()))
print(sum(A1['min_num_players'].isnull()))

# Do the same thing for B
em.set_property(B, 'fk_ltable', 'id')
em.set_property(B, 'fk_rtable', 'id')
em.set_property(B, 'ltable', B)
em.set_property(B, 'rtable', B)

B_all_attrs = list(B.columns.values)
# TODO: add 'min_age'
B_impute_attrs = ['year','min_num_players','max_num_players','min_gameplay_time','max_gameplay_time']
B_exclude_attrs = list(set(B_all_attrs) - set(B_impute_attrs))
B1 = em.impute_table(B, exclude_attrs=B_exclude_attrs, missing_val='NaN', strategy='most_frequent', axis=0, val_all_nans=0, verbose=True)

# Compare number of missing values to check the results
print(sum(B['min_num_players'].isnull()))
print(sum(B1['min_num_players'].isnull()))


65
0
5244
0

In [8]:
# Load the pre-labeled data
S = em.read_csv_metadata('sample_labeled.csv', 
                         key='_id',
                         ltable=A1, rtable=B1, 
                         fk_ltable='ltable_id', fk_rtable='rtable_id')


Metadata file is not present in the given path; proceeding to read the csv file.

In [9]:
# Split S into I an J
IJ = em.split_train_test(S, train_proportion=0.75, random_state=35)
I = IJ['train']
J = IJ['test']

In [10]:
corres = em.get_attr_corres(A1, B1)
print(corres)


{'rtable':           id  \
0          1   
1          2   
2          3   
3          4   
4          5   
5          6   
6          7   
7          8   
8          9   
9         10   
10        11   
11        12   
12        13   
13        14   
14        15   
15        16   
16        17   
17        18   
18        19   
19        20   
20        21   
21        22   
22        23   
23        24   
24        25   
25        26   
26        27   
27        28   
28        29   
29        30   
...      ...   
10266  10269   
10267  10270   
10268  10271   
10269  10272   
10270  10273   
10271  10274   
10272  10275   
10273  10276   
10274  10277   
10275  10278   
10276  10279   
10277  10280   
10278  10281   
10279  10282   
10280  10283   
10281  10284   
10282  10285   
10283  10286   
10284  10287   
10285  10288   
10286  10289   
10287  10290   
10288  10291   
10289  10292   
10290  10293   
10291  10294   
10292  10295   
10293  10296   
10294  10297   
10295  10298   

                                                                                           name  \
0                                           Panzer Exp. #3: Drive to the Rhine - The 2nd Front    
1                                                   Call of Cthulhu LCG: In the Dread of Night    
2                                                    Dungeons and Dragons: The Legend of Drizzt   
3                                                                   Istanbul: Mocha & Baksheesh   
4                                                                                 Perfect Alibi   
5                                                     Star Realms: Crisis – Fleets & Fortresses   
6                                                                                     Nexus Ops   
7                                                                              The King is Dead   
8                                           Pokemon TCG: XY Fates Collide Booster Display (36)    
9                                                   Tide of Iron: Designer Series - Volume One    
10                                                         Star Wars: The Card Game - So Be It    
11                                                                       Dead Men Tell No Tales   
12                                                                               Crown of Roses   
13                                                Shadows of Brimstone: Trun Hunters Enemy Pack   
14                                                  Civilization: Wisdom and Warfare Expansion    
15                                     Across the Aller - Last Stand in Northwest Europe, 1945    
16                                                                   Marbles: Electric Eel Set    
17                                                              Ticket to Ride: Heart of Africa   
18                                                                               Copper Country   
19                                                       Special Agent Meeple Poker Chips (20)    
20                                                        Summoner Wars: The Filth Faction Deck   
21                                                    Advanced Civilization - Assyria Counters    
22                                                    Reaper: Master Series Paint: Clear Green    
23                                                                    Prague: The Empty Triumph   
24                                                  Reaper: Master Series Paint: Flow Improver    
25                                                                                Panic Bundle    
26                                                                          Cleopatra's Caboose   
27                                                                                     Ali Baba   
28                                                               Folding Cribbage Travel Board    
29                                                                  The Curse of the Black Dice   
...                                                                                         ...   
10266                                                                             Dwarven Dive    
10267                                                     Island Fortress: 5-6 Player Expansion   
10268                                      Warmachine High Command: Faith & Fortune - Core Set    
10269                                                                                    Jaipur   
10270                                                                      Expo 1906 Board Game   
10271                                                                             Dungeon Lords   
10272                                                        Galaxy Defenders: Elite Alien Army   
10273                                 Summoner Wars: Jungle Elves Second Summoner Faction Deck    
10274                                                                    Frank's Zoo Card Game    
10275                                                          Clash of Cultures: Civilizations   
10276                                             Star Wars LCG: Jump to Lightspeed Force Pack    
10277                                                           Claustrophobia: Furor Sanguinis   
10278                Wings of Glory WWII: Rules and Accessories Plus Late Airplane Packs Combo    
10279                                              15" Walnut/Maple Chessboard w/1.75" Squares    
10280                          KV47 Recovery & Demolition Walker - Maksim/Marlen/Melor/Mikhail    
10281                                                      Myth: Orcneas Master of Masters Boss   
10282                                                         Mars Attacks: The Miniatures Game   
10283                                                           Fatal Alliances: The Great War    
10284                                                               Forge War: Second Printing    
10285                                                             Warmachine - Faith & Fortune    
10286                                                                                  Expedite   
10287                                                           Castle: The Detective Card Game   
10288                                                    Rum & Bones: Bone Devils Heroes Set #1   
10289                                                                            Eagle Kingdoms   
10290                                               Justine & Co. - Maintenance Contractors #2    
10291                                                 Shadowrun Crossfire DBG: Core Boxed Game    
10292                                                                         East is Red, The    
10293                                                                    Gunship: First Strike!   
10294                                                                               Extra Extra   
10295  Battles from the Age of Reason #2 - Zorndorf - Frederick vs. the Warriors of Holy Russia   

         year  rating  rank num_players  min_num_players  max_num_players  \
0      2015.0     NaN   NaN         NaN              2.0              4.0   
1      2015.0     NaN   NaN         NaN              2.0              4.0   
2      2011.0     NaN   NaN         1-5              1.0              5.0   
3      2015.0     NaN   NaN         2-5              2.0              5.0   
4      2015.0     NaN   NaN         3-5              3.0              5.0   
5      2014.0     NaN   NaN         2-6              2.0              6.0   
6      2005.0     NaN   NaN         2-4              2.0              4.0   
7      2015.0     NaN   NaN         2-4              2.0              4.0   
8      2015.0     NaN   NaN         NaN              2.0              4.0   
9      2015.0     NaN   NaN         NaN              2.0              4.0   
10     2015.0     NaN   NaN         NaN              2.0              4.0   
11     2015.0     NaN   NaN         2-5              2.0              5.0   
12     2012.0     NaN   NaN         2-4              2.0              4.0   
13     2015.0     NaN   NaN         1-6              1.0              6.0   
14     2015.0     NaN   NaN         NaN              2.0              4.0   
15     2015.0     NaN   NaN         NaN              2.0              4.0   
16     2015.0     NaN   NaN         NaN              2.0              4.0   
17     2012.0     NaN   NaN         2-5              2.0              5.0   
18     2015.0     NaN   NaN         2-4              2.0              4.0   
19     2015.0     NaN   NaN         NaN              2.0              4.0   
20     2012.0     NaN   NaN         2-4              2.0              4.0   
21     2015.0     NaN   NaN         NaN              2.0              4.0   
22     2015.0     NaN   NaN         NaN              2.0              4.0   
23     2014.0     NaN   NaN         1-4              1.0              4.0   
24     2015.0     NaN   NaN         NaN              2.0              4.0   
25     2015.0     NaN   NaN         NaN              2.0              4.0   
26     2010.0     NaN   NaN         3-5              3.0              5.0   
27     1974.0     NaN   NaN         2-4              2.0              4.0   
28     2015.0     NaN   NaN         NaN              2.0              4.0   
29     2015.0     NaN   NaN         2-4              2.0              4.0   
...       ...     ...   ...         ...              ...              ...   
10266  2015.0     NaN   NaN         NaN              2.0              4.0   
10267  2013.0     NaN   NaN         5-6              5.0              6.0   
10268  2015.0     NaN   NaN         NaN              2.0              4.0   
10269  2009.0     NaN   NaN           2              2.0              2.0   
10270  2015.0     NaN   NaN         1-4              1.0              4.0   
10271  2009.0     NaN   NaN         2-4              2.0              4.0   
10272  2014.0     NaN   NaN         1-5              1.0              5.0   
10273  2015.0     NaN   NaN         NaN              2.0              4.0   
10274  2015.0     NaN   NaN         NaN              2.0              4.0   
10275  2014.0     NaN   NaN         2-4              2.0              4.0   
10276  2015.0     NaN   NaN         NaN              2.0              4.0   
10277  2014.0     NaN   NaN         2-3              2.0              3.0   
10278  2015.0     NaN   NaN         NaN              2.0              4.0   
10279  2015.0     NaN   NaN         NaN              2.0              4.0   
10280  2015.0     NaN   NaN         NaN              2.0              4.0   
10281  2014.0     NaN   NaN         2-5              2.0              5.0   
10282  2014.0     NaN   NaN           2              2.0              2.0   
10283  2015.0     NaN   NaN         NaN              2.0              4.0   
10284  2015.0     NaN   NaN         NaN              2.0              4.0   
10285  2015.0     NaN   NaN         NaN              2.0              4.0   
10286  2012.0     NaN   NaN         2-4              2.0              4.0   
10287  2013.0     NaN   NaN         2-5              2.0              5.0   
10288  2015.0     NaN   NaN         2-6              2.0              6.0   
10289  1994.0     NaN   NaN         2-6              2.0              6.0   
10290  2015.0     NaN   NaN         NaN              2.0              4.0   
10291  2015.0     NaN   NaN         NaN              2.0              4.0   
10292  2015.0     NaN   NaN         NaN              2.0              4.0   
10293  2013.0     NaN   NaN         2-4              2.0              4.0   
10294  2015.0     NaN   NaN         2-6              2.0              6.0   
10295  1996.0     NaN   NaN         1-2              1.0              2.0   

      gameplay_time  min_gameplay_time  \
0               NaN               60.0   
1               NaN               60.0   
2        60 minutes               60.0   
3        60 minutes               60.0   
4        45 minutes               45.0   
5               NaN               60.0   
6        90 minutes               90.0   
7        50 minutes               50.0   
8               NaN               60.0   
9               NaN               60.0   
10              NaN               60.0   
11       75 minutes               75.0   
12      300 minutes              300.0   
13              NaN               60.0   
14              NaN               60.0   
15              NaN               60.0   
16              NaN               60.0   
17       60 minutes               60.0   
18       60 minutes               60.0   
19              NaN               60.0   
20       30 minutes               30.0   
21              NaN               60.0   
22              NaN               60.0   
23       30 minutes               30.0   
24              NaN               60.0   
25              NaN               60.0   
26      105 minutes              105.0   
27       20 minutes               20.0   
28              NaN               60.0   
29       20 minutes               20.0   
...             ...                ...   
10266           NaN               60.0   
10267   120 minutes              120.0   
10268           NaN               60.0   
10269    30 minutes               30.0   
10270    90 minutes               90.0   
10271    90 minutes               90.0   
10272   120 minutes              120.0   
10273           NaN               60.0   
10274           NaN               60.0   
10275   240 minutes              240.0   
10276           NaN               60.0   
10277    45 minutes               45.0   
10278           NaN               60.0   
10279           NaN               60.0   
10280           NaN               60.0   
10281   120 minutes              120.0   
10282   120 minutes              120.0   
10283           NaN               60.0   
10284           NaN               60.0   
10285           NaN               60.0   
10286    30 minutes               30.0   
10287    25 minutes               25.0   
10288           NaN               60.0   
10289    45 minutes               45.0   
10290           NaN               60.0   
10291           NaN               60.0   
10292           NaN               60.0   
10293    30 minutes               30.0   
10294   150 minutes              150.0   
10295   360 minutes              360.0   

                                                      ...                                                   \
0                                                     ...                                                    
1                                                     ...                                                    
2                                                     ...                                                    
3                                                     ...                                                    
4                                                     ...                                                    
5                                                     ...                                                    
6                                                     ...                                                    
7                                                     ...                                                    
8                                                     ...                                                    
9                                                     ...                                                    
10                                                    ...                                                    
11                                                    ...                                                    
12                                                    ...                                                    
13                                                    ...                                                    
14                                                    ...                                                    
15                                                    ...                                                    
16                                                    ...                                                    
17                                                    ...                                                    
18                                                    ...                                                    
19                                                    ...                                                    
20                                                    ...                                                    
21                                                    ...                                                    
22                                                    ...                                                    
23                                                    ...                                                    
24                                                    ...                                                    
25                                                    ...                                                    
26                                                    ...                                                    
27                                                    ...                                                    
28                                                    ...                                                    
29                                                    ...                                                    
...                                                   ...                                                    
10266                                                 ...                                                    
10267                                                 ...                                                    
10268                                                 ...                                                    
10269                                                 ...                                                    
10270                                                 ...                                                    
10271                                                 ...                                                    
10272                                                 ...                                                    
10273                                                 ...                                                    
10274                                                 ...                                                    
10275                                                 ...                                                    
10276                                                 ...                                                    
10277                                                 ...                                                    
10278                                                 ...                                                    
10279                                                 ...                                                    
10280                                                 ...                                                    
10281                                                 ...                                                    
10282                                                 ...                                                    
10283                                                 ...                                                    
10284                                                 ...                                                    
10285                                                 ...                                                    
10286                                                 ...                                                    
10287                                                 ...                                                    
10288                                                 ...                                                    
10289                                                 ...                                                    
10290                                                 ...                                                    
10291                                                 ...                                                    
10292                                                 ...                                                    
10293                                                 ...                                                    
10294                                                 ...                                                    
10295                                                 ...                                                    

       complexity_weight category  mechanisms  type  \
0                    NaN      NaN         NaN   NaN   
1                    NaN      NaN         NaN   NaN   
2                    NaN      NaN         NaN   NaN   
3                    NaN      NaN         NaN   NaN   
4                    NaN      NaN         NaN   NaN   
5                    NaN      NaN         NaN   NaN   
6                    NaN      NaN         NaN   NaN   
7                    NaN      NaN         NaN   NaN   
8                    NaN      NaN         NaN   NaN   
9                    NaN      NaN         NaN   NaN   
10                   NaN      NaN         NaN   NaN   
11                   NaN      NaN         NaN   NaN   
12                   NaN      NaN         NaN   NaN   
13                   NaN      NaN         NaN   NaN   
14                   NaN      NaN         NaN   NaN   
15                   NaN      NaN         NaN   NaN   
16                   NaN      NaN         NaN   NaN   
17                   NaN      NaN         NaN   NaN   
18                   NaN      NaN         NaN   NaN   
19                   NaN      NaN         NaN   NaN   
20                   NaN      NaN         NaN   NaN   
21                   NaN      NaN         NaN   NaN   
22                   NaN      NaN         NaN   NaN   
23                   NaN      NaN         NaN   NaN   
24                   NaN      NaN         NaN   NaN   
25                   NaN      NaN         NaN   NaN   
26                   NaN      NaN         NaN   NaN   
27                   NaN      NaN         NaN   NaN   
28                   NaN      NaN         NaN   NaN   
29                   NaN      NaN         NaN   NaN   
...                  ...      ...         ...   ...   
10266                NaN      NaN         NaN   NaN   
10267                NaN      NaN         NaN   NaN   
10268                NaN      NaN         NaN   NaN   
10269                NaN      NaN         NaN   NaN   
10270                NaN      NaN         NaN   NaN   
10271                NaN      NaN         NaN   NaN   
10272                NaN      NaN         NaN   NaN   
10273                NaN      NaN         NaN   NaN   
10274                NaN      NaN         NaN   NaN   
10275                NaN      NaN         NaN   NaN   
10276                NaN      NaN         NaN   NaN   
10277                NaN      NaN         NaN   NaN   
10278                NaN      NaN         NaN   NaN   
10279                NaN      NaN         NaN   NaN   
10280                NaN      NaN         NaN   NaN   
10281                NaN      NaN         NaN   NaN   
10282                NaN      NaN         NaN   NaN   
10283                NaN      NaN         NaN   NaN   
10284                NaN      NaN         NaN   NaN   
10285                NaN      NaN         NaN   NaN   
10286                NaN      NaN         NaN   NaN   
10287                NaN      NaN         NaN   NaN   
10288                NaN      NaN         NaN   NaN   
10289                NaN      NaN         NaN   NaN   
10290                NaN      NaN         NaN   NaN   
10291                NaN      NaN         NaN   NaN   
10292                NaN      NaN         NaN   NaN   
10293                NaN      NaN         NaN   NaN   
10294                NaN      NaN         NaN   NaN   
10295                NaN      NaN         NaN   NaN   

                                            BGG_link  \
0                                                NaN   
1                                                NaN   
2       http://www.boardgamegeek.com/boardgame/91872   
3      http://www.boardgamegeek.com/boardgame/172505   
4      http://www.boardgamegeek.com/boardgame/184085   
5      http://www.boardgamegeek.com/boardgame/166704   
6       http://www.boardgamegeek.com/boardgame/15363   
7      http://www.boardgamegeek.com/boardgame/172996   
8                                                NaN   
9                                                NaN   
10                                               NaN   
11     http://www.boardgamegeek.com/boardgame/141423   
12      http://www.boardgamegeek.com/boardgame/36480   
13     http://www.boardgamegeek.com/boardgame/181098   
14                                               NaN   
15                                               NaN   
16                                               NaN   
17     http://www.boardgamegeek.com/boardgame/131188   
18     http://www.boardgamegeek.com/boardgame/147206   
19                                               NaN   
20     http://www.boardgamegeek.com/boardgame/113225   
21                                               NaN   
22                                               NaN   
23      http://www.boardgamegeek.com/boardgame/39940   
24                                               NaN   
25                                               NaN   
26      http://www.boardgamegeek.com/boardgame/24795   
27      http://www.boardgamegeek.com/boardgame/10658   
28                                               NaN   
29     http://www.boardgamegeek.com/boardgame/170437   
...                                              ...   
10266                                            NaN   
10267  http://www.boardgamegeek.com/boardgame/113915   
10268                                            NaN   
10269   http://www.boardgamegeek.com/boardgame/54043   
10270  http://www.boardgamegeek.com/boardgame/180193   
10271   http://www.boardgamegeek.com/boardgame/45315   
10272  http://www.boardgamegeek.com/boardgame/149414   
10273                                            NaN   
10274                                            NaN   
10275  http://www.boardgamegeek.com/boardgame/146244   
10276                                            NaN   
10277  http://www.boardgamegeek.com/boardgame/162409   
10278                                            NaN   
10279                                            NaN   
10280                                            NaN   
10281  http://www.boardgamegeek.com/boardgame/154250   
10282  http://www.boardgamegeek.com/boardgame/147716   
10283                                            NaN   
10284                                            NaN   
10285                                            NaN   
10286  http://www.boardgamegeek.com/boardgame/122831   
10287  http://www.boardgamegeek.com/boardgame/131389   
10288  http://www.boardgamegeek.com/boardgame/184174   
10289    http://www.boardgamegeek.com/boardgame/1148   
10290                                            NaN   
10291                                            NaN   
10292                                            NaN   
10293  http://www.boardgamegeek.com/boardgame/109215   
10294  http://www.boardgamegeek.com/boardgame/128011   
10295   http://www.boardgamegeek.com/boardgame/11093   

                                                                                               store_names  \
0                                                                          Game Surplus,Noble Knight Games   
1                                                                              Great Boardgames,Amazon.com   
2      Miniature Market,Cardhaus,Cool Stuff Inc.,Roll for Crit,Amazon.com,Noble Knight Games,Boardlandi...   
3      Roll for Crit,Cardhaus,Cool Stuff Inc.,Miniature Market,Amazon.com,Time Well Spent Games,Funagai...   
4                                                                              Miniature Market,Amazon.com   
5      Roll for Crit,Miniature Market,Amazon.com,,BoardGameBliss,Great Boardgames,GeekStop Games,Amazon...   
6      Cool Stuff Inc.,BoardGameCo,Noble Knight Games,Amazon.com,Noble Knight Games,Noble Knight Games,...   
7      Cardhaus,Cool Stuff Inc.,Amazon.com,Funagain Games,Game Nerdz,Noble Knight Games,,Amazon.com,Gre...   
8                                                                                    Amazon.com,Game Nerdz   
9                                                                             Miniature Market,Amazon.com,   
10                                                                      Cardhaus,Amazon.com,Funagain Games   
11     Roll for Crit,Miniature Market,Cool Stuff Inc.,Cardhaus,Amazon.com,Noble Knight Games,,Starlit C...   
12      Cool Stuff Inc.,Amazon.com,Funagain Games,,Amazon.com,HitGames.ca,Shiregames,Amazon.com,Amazon.com   
13              Roll for Crit,Cool Stuff Inc.,Amazon.com,,HitGames.ca,BoardGameBliss,Amazon.com,Amazon.com   
14               Miniature Market,Cool Stuff Inc.,Amazon.com,,HitGames.ca,Amazon.com,Shiregames,Amazon.com   
15                                                                                      Noble Knight Games   
16                                                                               Funagain Games,Amazon.com   
17     Cool Stuff Inc.,Roll for Crit,Miniature Market,Game Surplus,Amazon.com,,GeekStop Games,BoardGame...   
18     Cardhaus,Amazon.com,Noble Knight Games,Funagain Games,Noble Knight Games,,BoardGameBliss,Amazon....   
19                                                                                        Miniature Market   
20                      Roll for Crit,Miniature Market,Cardhaus,Cool Stuff Inc.,Cool Stuff Inc.,Amazon.com   
21                                                                                     Noble Knight Games,   
22                                                                               Funagain Games,Amazon.com   
23                                                                             Cool Stuff Inc.,Amazon.com,   
24                                                                               Funagain Games,Amazon.com   
25                                                                                             HitGames.ca   
26                                 Amazon.com,Time Well Spent Games,Noble Knight Games,Noble Knight Games,   
27                                                       Amazon.com,Noble Knight Games,Noble Knight Games,   
28                                                        Starlit Citadel,Amazon.com,Amazon.com,Amazon.com   
29     Miniature Market,Amazon.com,Noble Knight Games,,Amazon.com,Starlit Citadel,BoardGameBliss,Amazon...   
...                                                                                                    ...   
10266                                                                                  Noble Knight Games,   
10267        Miniature Market,Cardhaus,Amazon.com,Noble Knight Games,,BoardGameBliss,Amazon.com,Amazon.com   
10268                                                                         Miniature Market,Amazon.com,   
10269  Miniature Market,Cool Stuff Inc.,Cardhaus,Cool Stuff Inc.,Roll for Crit,Amazon.com,Boardlandia,,...   
10270  Miniature Market,Cardhaus,Cool Stuff Inc.,Funagain Games,Noble Knight Games,Game Nerdz,,Great Bo...   
10271  Miniature Market,Roll for Crit,Cardhaus,Cool Stuff Inc.,Amazon.com,BoardGameCo,Noble Knight Game...   
10272  Cardhaus,Miniature Market,Roll for Crit,Cool Stuff Inc.,Game Surplus,Amazon.com,Game Nerdz,Funag...   
10273                            Roll for Crit,Cool Stuff Inc.,Miniature Market,Cool Stuff Inc.,Amazon.com   
10274                                                                           Cool Stuff Inc.,Amazon.com   
10275                                             Miniature Market,Cool Stuff Inc.,Game Surplus,Amazon.com   
10276                                               Cool Stuff Inc.,Miniature Market,Amazon.com,Game Nerdz   
10277  Cardhaus,Miniature Market,Cool Stuff Inc.,Noble Knight Games,Amazon.com,Funagain Games,,GeekStop...   
10278                                                                                      Cool Stuff Inc.   
10279                                                                                   Noble Knight Games   
10280                                                                        Amazon.com,Noble Knight Games   
10281  Cardhaus,Miniature Market,Amazon.com,Noble Knight Games,,BoardGameBliss,Great Boardgames,Amazon....   
10282                               Amazon.com,Game Nerdz,,BoardGameBliss,Amazon.com,Amazon.com,Amazon.com   
10283                        Cool Stuff Inc.,Noble Knight Games,Noble Knight Games,Amazon.com,,HitGames.ca   
10284                                                                     Amazon.com,Time Well Spent Games   
10285                                                                       Noble Knight Games,Amazon.com,   
10286                                                          Amazon.com,Noble Knight Games,Game Surplus,   
10287                                                                    HitGames.ca,Amazon.com,Amazon.com   
10288  Roll for Crit,Cardhaus,Cool Stuff Inc.,Miniature Market,Game Nerdz,Funagain Games,Amazon.com,,Gr...   
10289                                                                       Amazon.com,Noble Knight Games,   
10290                                                                Noble Knight Games,Noble Knight Games   
10291                                                                                           Game Nerdz   
10292                                 Amazon.com,Noble Knight Games,Noble Knight Games,Noble Knight Games,   
10293  Miniature Market,Cardhaus,Noble Knight Games,Amazon.com,BoardGameCo,Noble Knight Games,,HitGames...   
10294  Cardhaus,Miniature Market,Roll for Crit,Cool Stuff Inc.,Noble Knight Games,Amazon.com,Boardlandi...   
10295                                                                                   Noble Knight Games   

                                                                                              store_prices  \
0                                                                                            $39.95,$59.95   
1                                                                                    CAD 17.95,CAD  113.99   
2      $42.89,$42.99,$43.99,$43.99,$38.49,$40.00,$47.99,$51.95,$51.99,Prices from $ 30.00,CAD 66.95,CAD...   
3      $18.99,$19.49,$19.99,$20.09,$13.21,$22.50,$23.99,$24.95,$25.95,$27.47,Prices from $ 18.00,CAD 26...   
4                                                                                            $30.59,$19.99   
5      $3.99,$3.39,$15.86,Prices from $ 8.00,CAD 4.95,CAD 4.95,CAD 5.99,CAD  6.00,£3.99,£19.99,"EUR 21,89"   
6      $44.99,$28.49,$30.00,$39.95,$49.95,$85.00,$31.00,Prices from $ 27.00,CAD 39.99,CAD 55.99,CAD 59....   
7      $16.99,$16.99,$13.44,$19.99,$21.88,$21.95,Prices from $ 21.00,CAD  15.24,CAD 22.95,CAD 22.95,£9....   
8                                                                                            $94.95,$99.97   
9                                                                        $12.00,$14.24,Prices from $ 25.00   
10                                                                                     $11.21,$9.95,$14.95   
11     $28.99,$29.29,$30.49,$30.99,$31.85,$32.00,Prices from $ 1.00,CAD 47.99,CAD 39.95,CAD 39.95,CAD  ...   
12                 $52.99,$55.99,$62.99,Prices from $ 80.00,CAD  77.99,CAD 80.99,£59.99,£60.80,"EUR 19,95"   
13                          $12.99,$12.99,$13.58,Prices from $ 14.00,CAD 23.99,CAD 17.95,CAD  23.96,£15.43   
14                            $27.99,$29.99,$79.99,Prices from $ 199.00,CAD 48.99,CAD  58.59,£26.99,£27.86   
15                                                                                                  $74.95   
16                                                                                            $4.00,$15.00   
17     $18.79,$18.99,$20.00,$18.95,$24.99,Prices from $ 99.00,CAD 26.99,CAD 21.95,CAD 23.95,CAD 26.99,C...   
18     $38.99,$36.75,$45.00,$47.99,$49.95,Prices from $ 47.00,CAD 51.95,CAD  109.00,£44.99,£44.99,"EUR ...   
19                                                                                                   $4.00   
20                                                                     $5.99,$6.49,$6.50,$6.69,$6.69,$9.49   
21                                                                              $17.95,Prices from $ 17.00   
22                                                                                            $3.29,$37.00   
23                                                                       $62.99,$68.99,Prices from $ 85.00   
24                                                                                            $3.29,$12.94   
25                                                                                              CAD 101.99   
26                                                          $9.97,$38.95,$45.00,$49.95,Prices from $ 21.00   
27                                                                 $0.07,$10.00,$20.00,Prices from $ 22.00   
28                                                                   CAD 4.99,CAD  14.51,£7.99,"EUR 15,92"   
29       $34.19,$20.99,$51.95,Prices from $ 51.00,CAD  25.99,CAD 49.99,CAD 62.95,£16.72,£24.99,"EUR 18,99"   
...                                                                                                    ...   
10266                                                                             $6.49,Prices from $ 6.00   
10267                            $6.00,$11.40,$9.98,$17.95,Prices from $ 15.00,CAD 25.00,CAD  68.70,£66.50   
10268                                                                    $27.89,$22.49,Prices from $ 30.00   
10269  $21.29,$21.29,$21.49,$18.79,$18.99,$15.00,$19.99,Prices from $ 21.00,CAD 27.99,CAD 33.99,CAD 27....   
10270  $30.60,$35.99,$36.99,$35.99,$39.95,$43.80,Prices from $ 43.00,CAD 48.95,CAD 52.95,CAD  53.94,£34.74   
10271  $38.99,$41.99,$41.99,$43.99,$27.99,$40.00,$44.00,$45.00,$45.09,$49.95,$49.95,$51.99,$87.99,Price...   
10272  $13.99,$19.49,$19.99,$19.99,$9.95,$14.52,$21.18,$23.99,Prices from $ 20.00,CAD 22.95,CAD 23.95,C...   
10273                                                                        $5.99,$6.69,$6.49,$6.69,$7.81   
10274                                                                                         $8.19,$11.99   
10275                                                                         $32.49,$33.49,$29.50,$450.00   
10276                                                                           $11.29,$11.29,$6.28,$11.67   
10277  $22.99,$27.99,$29.79,$20.00,$23.08,$34.99,Prices from $ 26.00,CAD 34.99,CAD 38.95,CAD 38.95,CAD ...   
10278                                                                                               $46.99   
10279                                                                                               $35.95   
10280                                                                                        $20.47,$37.95   
10281      $16.99,$16.00,$8.49,$21.95,Prices from $ 25.00,CAD 7.95,CAD 21.95,CAD  51.37,£85.32,"EUR 46,23"   
10282                           $47.75,$60.47,Prices from $ 60.00,CAD 63.95,CAD  178.70,£31.22,"EUR 36,57"   
10283                                       $101.49,$95.00,$113.95,$124.99,Prices from $ 124.00,CAD 144.99   
10284                                                                                        $44.99,$52.95   
10285                                                                    $40.95,$69.28,Prices from $ 11.00   
10286                                                              $0.01,$50.00,$32.95,Prices from $ 25.00   
10287                                                                           CAD 29.99,CAD  43.99,£9.99   
10288  $14.99,$15.99,$15.99,$15.99,$15.99,$15.99,$57.64,Prices from $ 14.00,CAD 17.95,CAD 18.95,CAD 18....   
10289                                                                      $7.96,$8.00,Prices from $ 11.00   
10290                                                                                        $33.95,$83.95   
10291                                                                                               $45.83   
10292                                                       $9.99,$18.00,$22.00,$99.95,Prices from $ 15.00   
10293  $10.00,$23.20,$15.00,$15.78,$18.21,$29.95,Prices from $ 26.00,CAD 47.99,CAD  126.54,£20.99,"EUR ...   
10294  $58.50,$58.50,$58.99,$58.50,$10.95,$32.69,$58.50,$58.50,$58.99,Prices from $ 1.00,CAD  29.97,CAD...   
10295                                                                                              $125.00   

                                                                                              links_to_buy  \
0      http://www.gamesurplus.com/product_info.php?products_id=3281&utm_source=boardgameprices.com&utm_...   
1      http://www.greatboardgames.ca/call-of-cthulhu-lcg-in-the-dread-of-night.html?utm_source=boardgam...   
2      http://www.miniaturemarket.com/woc35594.html?utm_source=boardgameprices.com&utm_medium=feed&utm_...   
3      http://www.rollforcrit.com/product/istanbul-mocha-baksheesh/?utm_source=boardgameprices.com&utm_...   
4      http://www.miniaturemarket.com/ltp0555.html?utm_source=boardgameprices.com&utm_medium=feed&utm_c...   
5      http://www.rollforcrit.com/product/star-realms-crisis-fleets-fortresses/?utm_source=boardgamepri...   
6      http://www.coolstuffinc.com/p/137100?utm_source=boardgameprices.com&utm_medium=price-search&utm_...   
7      http://www.cardhaus.com/catalog/board_games/the_king_is_dead/339513?utm_source=boardgameprices.c...   
8      http://www.amazon.com/Pok%C3%A9mon-TCG-Collide-Booster-Display/dp/B01AAUSKXS%3FSubscriptionId%3D...   
9      http://www.miniaturemarket.com/1agtd04.html?utm_source=boardgameprices.com&utm_medium=feed&utm_c...   
10     http://www.cardhaus.com/catalog/board_games/star_wars_the_card_game__so_be_it/353453?utm_source=...   
11     http://www.rollforcrit.com/product/dead-men-tell-no-tales/?utm_source=boardgameprices.com&utm_me...   
12     http://www.coolstuffinc.com/p/165457?utm_source=boardgameprices.com&utm_medium=price-search&utm_...   
13     http://www.rollforcrit.com/product/shadows-of-brimstone-trun-hunters-enemy-pack/?utm_source=boar...   
14     http://www.miniaturemarket.com/ffgci03.html?utm_source=boardgameprices.com&utm_medium=feed&utm_c...   
15     http://www.nobleknight.com/ProductDetailSearch.asp_Q_ProductID_E_2147594596_A_InventoryID_E_2148...   
16     https://www.funagain.com/control/product?product_id=022368&affil=BGPC&utm_source=boardgameprices...   
17     http://www.coolstuffinc.com/p/165402?utm_source=boardgameprices.com&utm_medium=price-search&utm_...   
18     http://www.cardhaus.com/catalog/board_games/copper_country/307123?utm_source=boardgameprices.com...   
19     http://www.miniaturemarket.com/gps-sachips.html?utm_source=boardgameprices.com&utm_medium=feed&u...   
20     http://www.rollforcrit.com/product/summoner-wars-the-filth-faction-deck/?utm_source=boardgamepri...   
21     http://www.nobleknight.com/ProductDetailSearch.asp_Q_ProductID_E_2147575381_A_InventoryID_E_2148...   
22     https://www.funagain.com/control/product?product_id=035902&affil=BGPC&utm_source=boardgameprices...   
23     http://www.coolstuffinc.com/p/203498?utm_source=boardgameprices.com&utm_medium=price-search&utm_...   
24     https://www.funagain.com/control/product?product_id=035915&affil=BGPC&utm_source=boardgameprices...   
25     http://www.hitgames.ca/panic-bundle.html?utm_source=boardgameprices.com&utm_medium=price-search&...   
26     http://www.amazon.com/Z-Man-Games-Cleopatras-Caboose/dp/B0047UH2T6%3FSubscriptionId%3DAKIAJ4BCA4...   
27     http://www.amazon.com/Force-Will-Earnest-Worker-MPR-056/dp/B013538VAE%3FSubscriptionId%3DAKIAJ4B...   
28     http://www.starlitcitadel.com/games/folding-cribbage-travel-board.html?utm_source=boardgameprice...   
29     http://www.miniaturemarket.com/badcotbd.html?utm_source=boardgameprices.com&utm_medium=feed&utm_...   
...                                                                                                    ...   
10266  http://www.nobleknight.com/ProductDetailSearch.asp_Q_ProductID_E_2147626704_A_InventoryID_E_2148...   
10267  http://www.miniaturemarket.com/gsuh5051.html?utm_source=boardgameprices.com&utm_medium=feed&utm_...   
10268  http://www.miniaturemarket.com/pip61020.html?utm_source=boardgameprices.com&utm_medium=feed&utm_...   
10269  http://www.miniaturemarket.com/asmjai01.html?utm_source=boardgameprices.com&utm_medium=feed&utm_...   
10270  http://www.miniaturemarket.com/gth6143.html?utm_source=boardgameprices.com&utm_medium=feed&utm_c...   
10271  http://www.miniaturemarket.com/cge00007.html?utm_source=boardgameprices.com&utm_medium=feed&utm_...   
10272  http://www.cardhaus.com/catalog/board_games/galaxy_defenders_elite_alien_army/254113?utm_source=...   
10273  http://www.rollforcrit.com/product/summoner-wars-jungle-elves-second-summoner-faction-deck/?utm_...   
10274  http://www.coolstuffinc.com/p/135707?utm_source=boardgameprices.com&utm_medium=price-search&utm_...   
10275  http://www.miniaturemarket.com/zmg71001.html?utm_source=boardgameprices.com&utm_medium=feed&utm_...   
10276  http://www.coolstuffinc.com/p/210720?utm_source=boardgameprices.com&utm_medium=price-search&utm_...   
10277  http://www.cardhaus.com/catalog/board_games/claustrophobia_furor_sanguinis/301603?utm_source=boa...   
10278  http://www.coolstuffinc.com/p/207562?utm_source=boardgameprices.com&utm_medium=price-search&utm_...   
10279  http://www.nobleknight.com/ProductDetailSearch.asp_Q_ProductID_E_2147533169_A_InventoryID_E_2148...   
10280  http://www.amazon.com/Dust-Tactics-Recovery-Demolition-Walker/dp/1616616180%3FSubscriptionId%3DA...   
10281  http://www.cardhaus.com/catalog/board_games/myth_orcneas_master_of_masters_boss/301453?utm_sourc...   
10282  http://www.amazon.com/Mars-Attacks-The-Miniatures-Game/dp/B00O2785VI%3FSubscriptionId%3DAKIAJ4BC...   
10283  http://www.coolstuffinc.com/p/225335?utm_source=boardgameprices.com&utm_medium=price-search&utm_...   
10284  http://www.amazon.com/Cephalofair-Games-CPH0102-Forge-Game/dp/B017XTDL6K%3FSubscriptionId%3DAKIA...   
10285  http://www.nobleknight.com/ProductDetailSearch.asp_Q_ProductID_E_2147558615_A_InventoryID_E_2148...   
10286  http://www.amazon.com/Magic-Gathering-Expedite-Oath-Gatewatch/dp/B01AASV54G%3FSubscriptionId%3DA...   
10287  http://www.hitgames.ca/castle-the-detective-card-game.html?utm_source=boardgameprices.com&utm_me...   
10288  http://www.rollforcrit.com/product/rum-bones-bone-devils-heroes-set-1/?utm_source=boardgameprice...   
10289  http://www.amazon.com/Douglas-Cuddle-Toys-3747-Colbert/dp/B000Q6QJG2%3FSubscriptionId%3DAKIAJ4BC...   
10290  http://www.nobleknight.com/ProductDetailSearch.asp_Q_ProductID_E_2147554464_A_InventoryID_E_2148...   
10291  http://www.gamenerdz.com/shadowrun-crossfire-dbg-core-boxed-game?utm_source=boardgameprices.com&...   
10292  http://www.amazon.com/Paw-Patrol-Ryders-Pup-Pad/dp/B00NJVHB8Q%3FSubscriptionId%3DAKIAJ4BCA4P5VKV...   
10293  http://www.miniaturemarket.com/epg84350.html?utm_source=boardgameprices.com&utm_medium=feed&utm_...   
10294  http://www.cardhaus.com/catalog/board_games/extra_extra/330803?utm_source=boardgameprices.com&ut...   
10295  http://www.nobleknight.com/ProductDetailSearch.asp_Q_ProductID_E_1761287263_A_InventoryID_E_2148...   

                                                                                              availability  \
0                                                                                        In Stock,In Stock   
1                                                                                        In Stock,In Stock   
2      In Stock,In Stock,In Stock,Unavailable,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In ...   
3      In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Sto...   
4                                                                                        In Stock,In Stock   
5      In Stock,Unavailable,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In ...   
6      Unavailable,In Stock,In Stock,In Stock,In Stock,In Stock,Unavailable,In Stock,In Stock,On the Wa...   
7      In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Sto...   
8                                                                                     In Stock,Unavailable   
9                                                                               In Stock,In Stock,In Stock   
10                                                                              In Stock,In Stock,In Stock   
11     In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,Unavailable,Unavailable,...   
12                        In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock   
13                              In Stock,In Stock,In Stock,In Stock,In Stock,Unavailable,In Stock,In Stock   
14                     Unavailable,Unavailable,In Stock,In Stock,Unavailable,In Stock,Unavailable,In Stock   
15                                                                                                In Stock   
16                                                                                       In Stock,In Stock   
17     Unavailable,Unavailable,Unavailable,Unavailable,In Stock,In Stock,On the Way,Unavailable,Unavail...   
18      In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock   
19                                                                                                In Stock   
20                                                In Stock,In Stock,In Stock,In Stock,Unavailable,In Stock   
21                                                                                       In Stock,In Stock   
22                                                                                       In Stock,In Stock   
23                                                                           Unavailable,In Stock,In Stock   
24                                                                                       In Stock,In Stock   
25                                                                                             Unavailable   
26                                                            In Stock,In Stock,In Stock,In Stock,In Stock   
27                                                                     In Stock,In Stock,In Stock,In Stock   
28                                                                     In Stock,In Stock,In Stock,In Stock   
29       On the Way,In Stock,In Stock,In Stock,In Stock,Unavailable,Unavailable,In Stock,In Stock,In Stock   
...                                                                                                    ...   
10266                                                                                    In Stock,In Stock   
10267                              In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock   
10268                                                                           In Stock,In Stock,In Stock   
10269  In Stock,In Stock,In Stock,Unavailable,Unavailable,In Stock,In Stock,In Stock,On the Way,In Stoc...   
10270   In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock   
10271  In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Sto...   
10272  In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Sto...   
10273                                                   In Stock,In Stock,Unavailable,Unavailable,In Stock   
10274                                                                                    In Stock,In Stock   
10275                                                         Unavailable,Unavailable,Unavailable,In Stock   
10276                                                                  In Stock,In Stock,In Stock,In Stock   
10277  In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,Unavai...   
10278                                                                                          Unavailable   
10279                                                                                             In Stock   
10280                                                                                    In Stock,In Stock   
10281   In Stock,Unavailable,In Stock,In Stock,In Stock,Unavailable,Unavailable,In Stock,In Stock,In Stock   
10282                                    In Stock,In Stock,In Stock,Unavailable,In Stock,In Stock,In Stock   
10283                                                In Stock,In Stock,In Stock,In Stock,In Stock,In Stock   
10284                                                                                    In Stock,In Stock   
10285                                                                           In Stock,In Stock,In Stock   
10286                                                               In Stock,In Stock,Unavailable,In Stock   
10287                                                                        Unavailable,In Stock,In Stock   
10288  In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Sto...   
10289                                                                           In Stock,In Stock,In Stock   
10290                                                                                    In Stock,In Stock   
10291                                                                                          Unavailable   
10292                                                         In Stock,In Stock,In Stock,In Stock,In Stock   
10293  In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,Unavailable,In Stock,In Stock,In ...   
10294  In Stock,In Stock,In Stock,Unavailable,In Stock,In Stock,In Stock,In Stock,In Stock,In Stock,In ...   
10295                                                                                             In Stock   

                                                                                       international_store  
0                                                                                              False,False  
1                                                                                                True,True  
2      False,False,False,False,False,False,False,False,False,False,True,True,True,True,True,True,True,True  
3      False,False,False,False,False,False,False,False,False,False,False,True,True,True,True,True,True,...  
4                                                                                              False,False  
5                                               False,False,False,False,True,True,True,True,True,True,True  
6             False,False,False,False,False,False,False,False,True,True,True,True,True,True,True,True,True  
7                                  False,False,False,False,False,False,False,True,True,True,True,True,True  
8                                                                                              False,False  
9                                                                                        False,False,False  
10                                                                                       False,False,False  
11                            False,False,False,False,False,False,False,True,True,True,True,True,True,True  
12                                                        False,False,False,False,True,True,True,True,True  
13                                                             False,False,False,False,True,True,True,True  
14                                                             False,False,False,False,True,True,True,True  
15                                                                                                   False  
16                                                                                             False,False  
17                        False,False,False,False,False,False,True,True,True,True,True,True,True,True,True  
18                                            False,False,False,False,False,False,True,True,True,True,True  
19                                                                                                   False  
20                                                                     False,False,False,False,False,False  
21                                                                                             False,False  
22                                                                                             False,False  
23                                                                                       False,False,False  
24                                                                                             False,False  
25                                                                                                    True  
26                                                                           False,False,False,False,False  
27                                                                                 False,False,False,False  
28                                                                                     True,True,True,True  
29                                                   False,False,False,False,True,True,True,True,True,True  
...                                                                                                    ...  
10266                                                                                          False,False  
10267                                                         False,False,False,False,False,True,True,True  
10268                                                                                    False,False,False  
10269    False,False,False,False,False,False,False,False,True,True,True,True,True,True,True,True,True,True  
10270                                        False,False,False,False,False,False,False,True,True,True,True  
10271  False,False,False,False,False,False,False,False,False,False,False,False,False,False,True,True,Tr...  
10272             False,False,False,False,False,False,False,False,False,True,True,True,True,True,True,True  
10273                                                                        False,False,False,False,False  
10274                                                                                          False,False  
10275                                                                              False,False,False,False  
10276                                                                              False,False,False,False  
10277                    False,False,False,False,False,False,False,True,True,True,True,True,True,True,True  
10278                                                                                                False  
10279                                                                                                False  
10280                                                                                          False,False  
10281                                               False,False,False,False,False,True,True,True,True,True  
10282                                                                False,False,False,True,True,True,True  
10283                                                                   False,False,False,False,False,True  
10284                                                                                          False,False  
10285                                                                                    False,False,False  
10286                                                                              False,False,False,False  
10287                                                                                       True,True,True  
10288                        False,False,False,False,False,False,False,False,True,True,True,True,True,True  
10289                                                                                    False,False,False  
10290                                                                                          False,False  
10291                                                                                                False  
10292                                                                        False,False,False,False,False  
10293                                        False,False,False,False,False,False,False,True,True,True,True  
10294            False,False,False,False,False,False,False,False,False,False,True,True,True,True,True,True  
10295                                                                                                False  

[10296 rows x 22 columns], 'ltable':         id                                               name    year  rating  \
0        1                                  Sweet Valley High  1988.0     4.8   
1        2                007 James Bond: You Only Live Twice  1985.0     5.7   
2        3                           Rock Paper Scissors Inc.  2003.0     5.0   
3        4                                         VisualEyes  2003.0     5.6   
4        5                         007 James Bond: Goldfinger  1985.0     6.0   
5        6                                      Assembly Line  2003.0     5.0   
6        7                                           Cronberg  2003.0     6.0   
7        8                                        Münchhausen  1996.0     5.9   
8        9                                    Lickin' Lizards  1993.0     5.4   
9       10                                     Slobberin' Sam  1993.0     2.0   
10      11                              The Odd-Ball Scenario  1996.0     5.0   
11      12                            Husch Husch kleine Hexe  1994.0     5.5   
12      13                                          Mad Dash!  1991.0     5.4   
13      14                                             Batalo  1992.0     6.2   
14      15                                             Tilez!  1997.0     4.9   
15      16                                       Blaster Bots  2003.0     5.5   
16      17                             Electronic Dream Phone  1991.0     5.6   
17      18                              World Cup in the Void  2003.0     NaN   
18      19                                             Sabuca  1999.0     6.0   
19      20                           Korea: The Forgotten War  2003.0     8.1   
20      21                                    Let's Do Lunch!  2000.0     5.3   
21      22              Kung Fu Samurai on Giant Robot Island  2003.0     5.8   
22      23                 Grave Robbers II: Skippy's Revenge  2003.0     6.2   
23      24                                              Booty  1995.0     3.9   
24      25                               Lights Camera Action  2003.0     7.5   
25      26                                             Coodju  2001.0     5.1   
26      27                                       Radar Search  1969.0     4.6   
27      28                                           Stampin'  1989.0     4.4   
28      29                                           Glasnost  1989.0     4.4   
29      30                                   Wall Street Game  1998.0     3.0   
...    ...                                                ...     ...     ...   
6961  6962                                Melvin the Moon Man  1959.0     NaN   
6962  6963                                          Le Camino  1973.0     6.3   
6963  6964                          Super Nova: Event Horizon  2000.0     5.0   
6964  6965                     Deadlands: Doomtown Range Wars  2000.0     6.3   
6965  6966                                      Backscrabbing  1989.0     3.5   
6966  6967                                        Air & Armor  1986.0     7.6   
6967  6968                                   Destination Mars  1999.0     5.1   
6968  6969                                       Star Empires  1987.0     5.8   
6969  6970                                   Echelons of Fury  1995.0     4.5   
6970  6971                                              Force  1977.0     4.4   
6971  6972                                         Off Course  1986.0     7.0   
6972  6973                                 Fields of Chivalry  2002.0     5.0   
6973  6974                                        Shipwrecked  2000.0     5.7   
6974  6975                    East & West: Global War in 1948  2000.0     6.6   
6975  6976                  Captain Tsubasa Trading Card Game  2002.0     5.8   
6976  6977                                     Käp'tän Buddel  1989.0     3.0   
6977  6978                                 Abaco Zuzzurellone  1995.0     5.6   
6978  6979                                          Top Quark  1996.0     5.7   
6979  6980                                       Time Control  2003.0     2.1   
6980  6981  Murder Mystery Party: A Taste for Wine and Murder  1997.0     6.8   
6981  6982         Murder à la carte: Pasta Passion & Pistols  1995.0     5.8   
6982  6983                                           Bosworth  1998.0     5.8   
6983  6984            Murder à la carte: Bullets 'N' Barbecue  1994.0     4.9   
6984  6985  Murder Mystery Party: Murder on the S. S. Titania  1996.0     6.0   
6985  6986                                       Floß geht's!  2003.0     6.7   
6986  6987                                        Doktor Igel  2001.0     5.7   
6987  6988                                         Fleet Fins  2003.0     6.4   
6988  6989                                Audit: The Tax Game  1979.0     NaN   
6989  6990                        Lodz 1914: First Blitzkrieg  1998.0     6.9   
6990  6991                                           Hegemony  2003.0     5.5   

         rank num_players  min_num_players  max_num_players gameplay_time  \
0     11444.0         2-4              2.0              4.0       30  Min   
1         NaN           2              2.0              2.0       20  Min   
2         NaN           2              2.0              2.0       10  Min   
3      8784.0         2-8              2.0              8.0       30  Min   
4         NaN           2              2.0              2.0       20  Min   
5         NaN           2              2.0              2.0       20  Min   
6      4074.0         2-4              2.0              4.0       20  Min   
7      8152.0         3-8              3.0              8.0       30  Min   
8         NaN         2-4              2.0              4.0       20  Min   
9         NaN         2-4              2.0              4.0       20  Min   
10        NaN         2-6              2.0              6.0       30  Min   
11    10275.0         2-6              2.0              6.0       15  Min   
12        NaN           2              2.0              2.0       10  Min   
13        NaN           2              2.0              2.0       30  Min   
14        NaN         2-4              2.0              4.0       30  Min   
15        NaN         2-6              2.0              6.0       30  Min   
16    10924.0         1-4              1.0              4.0       20  Min   
17        NaN           2              2.0              2.0       20  Min   
18        NaN         2-6              2.0              6.0       20  Min   
19     1906.0           2              2.0              2.0      360  Min   
20    10659.0           2              2.0              2.0       20  Min   
21     6690.0         2-6              2.0              6.0       30  Min   
22     4266.0         2-6              2.0              6.0       30  Min   
23        NaN         2-4              2.0              4.0       45  Min   
24        NaN           2              2.0              2.0       45  Min   
25        NaN           4              4.0              4.0       30  Min   
26        NaN           2              2.0              2.0       30  Min   
27        NaN         2-4              2.0              4.0       60  Min   
28        NaN         2-8              2.0              8.0       90  Min   
29        NaN         2-5              2.0              5.0       60  Min   
...       ...         ...              ...              ...           ...   
6961      NaN         2-6              2.0              6.0       30  Min   
6962      NaN         2-4              2.0              4.0       45  Min   
6963      NaN         2-6              2.0              6.0       90  Min   
6964   5171.0         2-8              2.0              8.0       90  Min   
6965      NaN         4-6              4.0              6.0       90  Min   
6966   3295.0           2              2.0              2.0      120  Min   
6967      NaN           2              2.0              2.0       20  Min   
6968      NaN         2-6              2.0              6.0       60  Min   
6969  12018.0           2              2.0              2.0       60  Min   
6970      NaN           2              2.0              2.0       20  Min   
6971      NaN         1-6              1.0              6.0       45  Min   
6972      NaN         2-4              2.0              4.0       60  Min   
6973   6468.0         2-4              2.0              4.0       30  Min   
6974      NaN         2-4              2.0              4.0      300  Min   
6975      NaN           2              2.0              2.0       60  Min   
6976      NaN         2-4              2.0              4.0       45  Min   
6977      NaN           2              2.0              2.0       45  Min   
6978      NaN           2              2.0              2.0       30  Min   
6979  12584.0         2-4              2.0              4.0       45  Min   
6980      NaN         6-8              6.0              8.0      180  Min   
6981      NaN         6-8              6.0              8.0      180  Min   
6982   5871.0         2-4              2.0              4.0       60  Min   
6983      NaN           6              6.0              6.0      180  Min   
6984      NaN           8              8.0              8.0      240  Min   
6985      NaN         2-4              2.0              4.0       10  Min   
6986  10019.0         2-5              2.0              5.0       10  Min   
6987      NaN         2-4              2.0              4.0       10  Min   
6988      NaN         2-6              2.0              6.0       90  Min   
6989      NaN         2-4              2.0              4.0      360  Min   
6990      NaN         2-4              2.0              4.0       60  Min   

      min_gameplay_time         ...           complexity_weight  \
0                  30.0         ...                        1.00   
1                  20.0         ...                        1.33   
2                  10.0         ...                        1.50   
3                  30.0         ...                        1.41   
4                  20.0         ...                        1.50   
5                  20.0         ...                        1.50   
6                  20.0         ...                        1.91   
7                  30.0         ...                        1.09   
8                  20.0         ...                         NaN   
9                  20.0         ...                         NaN   
10                 30.0         ...                         NaN   
11                 15.0         ...                        1.00   
12                 10.0         ...                        1.00   
13                 30.0         ...                        2.00   
14                 30.0         ...                        1.50   
15                 30.0         ...                         NaN   
16                 20.0         ...                        2.33   
17                 20.0         ...                         NaN   
18                 20.0         ...                        1.33   
19                360.0         ...                        4.21   
20                 20.0         ...                        1.14   
21                 30.0         ...                        1.50   
22                 30.0         ...                        1.28   
23                 45.0         ...                        1.50   
24                 45.0         ...                         NaN   
25                 30.0         ...                        2.00   
26                 30.0         ...                        1.20   
27                 60.0         ...                         NaN   
28                 90.0         ...                        2.00   
29                 60.0         ...                         NaN   
...                 ...         ...                         ...   
6961               30.0         ...                        1.00   
6962               45.0         ...                        2.00   
6963               90.0         ...                        1.00   
6964               90.0         ...                        2.20   
6965               90.0         ...                        3.00   
6966              120.0         ...                        3.56   
6967               20.0         ...                        1.33   
6968               60.0         ...                        3.00   
6969               60.0         ...                        2.20   
6970               20.0         ...                        1.00   
6971               45.0         ...                        2.00   
6972               60.0         ...                         NaN   
6973               30.0         ...                        1.48   
6974              300.0         ...                        2.50   
6975               60.0         ...                         NaN   
6976               45.0         ...                         NaN   
6977               45.0         ...                         NaN   
6978               30.0         ...                         NaN   
6979               45.0         ...                        2.50   
6980              180.0         ...                         NaN   
6981              180.0         ...                        2.33   
6982               60.0         ...                        2.10   
6983              180.0         ...                        1.00   
6984              240.0         ...                         NaN   
6985               10.0         ...                        1.00   
6986               10.0         ...                        1.00   
6987               10.0         ...                        1.00   
6988               90.0         ...                         NaN   
6989              360.0         ...                        4.00   
6990               60.0         ...                         NaN   

                                                                  category  \
0                                                       Memory;Novel-based   
1                          Adventure;Movies / TV / Radio theme;Novel-based   
2                                           Bluffing;Economic;Print & Play   
3                   Dice;Negotiation;Party Game;Puzzle;Real-time;Word Game   
4                          Adventure;Movies / TV / Radio theme;Novel-based   
5                         Abstract Strategy;Bluffing;Economic;Print & Play   
6                                               City Building;Print & Play   
7                                               Bluffing;Card Game;Fantasy   
8                     Action / Dexterity;Animals;Children's Game;Real-time   
9                    Action / Dexterity;Animals;Children's Game;Electronic   
10                             Expansion for Base-game;Spies/Secret Agents   
11                              Children's Game;Dice;Fantasy;Memory;Racing   
12                                            Card Game;Educational;Racing   
13                                                       Abstract Strategy   
14                                                       Abstract Strategy   
15                                      Card Game;Fighting;Science Fiction   
16                                    Children's Game;Deduction;Electronic   
17                                                        Card Game;Sports   
18                                                     Card Game;Word Game   
19                                       Korean War;Modern Warfare;Wargame   
20                                      Abstract Strategy;Dice;Prehistoric   
21               Card Game;Humor;Movies / TV / Radio theme;Science Fiction   
22        Card Game;Horror;Humor;Movies / TV / Radio theme;Science Fiction   
23                                           Abstract Strategy;Maze;Racing   
24                             Movies / TV / Radio theme;Party Game;Trivia   
25                                                    Party Game;Word Game   
26                       Bluffing;Deduction;Electronic;Spies/Secret Agents   
27                                                                     NaN   
28                                                               Political   
29                                                                Economic   
...                                                                    ...   
6961                                     Science Fiction;Space Exploration   
6962                                                     Abstract Strategy   
6963  Card Game;Economic;Expansion for Base-game;Political;Science Fiction   
6964                          American West;Collectible Components;Fantasy   
6965                         Economic;Industry / Manufacturing;Negotiation   
6966                                                Modern Warfare;Wargame   
6967                                      Card Game;Racing;Science Fiction   
6968                                             Card Game;Science Fiction   
6969                         Card Game;Collectible Components;World War II   
6970                                  Abstract Strategy;Action / Dexterity   
6971                                                           Dice;Sports   
6972                                                Dice;Fighting;Medieval   
6973                                                             Card Game   
6974                                                  Wargame;World War II   
6975                                   Card Game;Comic Book / Strip;Sports   
6976                                                               Pirates   
6977                                                             Word Game   
6978                                                     Abstract Strategy   
6979                                             Real-time;Science Fiction   
6980                                   Deduction;Murder/Mystery;Party Game   
6981                                   Deduction;Murder/Mystery;Party Game   
6982                                                     Abstract Strategy   
6983                                   Deduction;Murder/Mystery;Party Game   
6984                                   Deduction;Murder/Mystery;Party Game   
6985                                       Children's Game;Nautical;Racing   
6986                       Action / Dexterity;Animals;Children's Game;Dice   
6987            Action / Dexterity;Animals;Children's Game;Memory;Nautical   
6988                                                              Economic   
6989                                                   Wargame;World War I   
6990                                                     Abstract Strategy   

                                                                                 mechanisms  \
0                                                Memory;Roll / Spin and Move;Set Collection   
1                                                                      Roll / Spin and Move   
2                           Rock-Paper-Scissors;Simultaneous Action Selection;Stock Holding   
3                                                      Dice Rolling;Paper-and-Pencil;Voting   
4                                                        Area Movement;Roll / Spin and Move   
5                                         Rock-Paper-Scissors;Simultaneous Action Selection   
6                                                             Area Enclosure;Tile Placement   
7                                                                                       NaN   
8                                                                                       NaN   
9                                                                                       NaN   
10                                                                                      NaN   
11                                                 Dice Rolling;Memory;Roll / Spin and Move   
12                                                         Hand Management;Pattern Building   
13                                                                            Area Movement   
14                                                                           Tile Placement   
15                                                                             Dice Rolling   
16                                                                     Roll / Spin and Move   
17                                                        Simulation;Variable Player Powers   
18                                                                          Hand Management   
19                                                  Dice Rolling;Hex-and-Counter;Simulation   
20                                                                             Dice Rolling   
21                                                                                      NaN   
22                                                                                      NaN   
23                                          Dice Rolling;Pick-up and Deliver;Tile Placement   
24                                                           Co-operative Play;Role Playing   
25                                                                                      NaN   
26                    Point to Point Movement;Secret Unit Deployment;Variable Player Powers   
27                                      Auction/Bidding;Roll / Spin and Move;Set Collection   
28                                                                                      NaN   
29                                                       Roll / Spin and Move;Stock Holding   
...                                                                                     ...   
6961                                                                           Dice Rolling   
6962                                                                         Tile Placement   
6963                                        Hand Management;Point to Point Movement;Trading   
6964                                                                                    NaN   
6965       Commodity Speculation;Hand Management;Roll / Spin and Move;Stock Holding;Trading   
6966                               Action Point Allowance System;Hex-and-Counter;Simulation   
6967                                                                         Set Collection   
6968                                                                         Set Collection   
6969                                          Campaign / Battle Card Driven;Hand Management   
6970                                                                 Route/Network Building   
6971                              Card Drafting;Dice Rolling;Modular Board;Paper-and-Pencil   
6972  Campaign / Battle Card Driven;Dice Rolling;Rock-Paper-Scissors;Variable Player Powers   
6973                                                                        Auction/Bidding   
6974                                                                          Area Movement   
6975                                                    Role Playing;Variable Player Powers   
6976                                             Action Point Allowance System;Dice Rolling   
6977                                                                       Paper-and-Pencil   
6978                                                                          Area Movement   
6979                          Auction/Bidding;Hand Management;Simultaneous Action Selection   
6980                                                                                 Acting   
6981                                                                                 Acting   
6982                                                                          Grid Movement   
6983                                                                                 Acting   
6984                                                                                 Acting   
6985                    Dice Rolling;Modular Board;Pattern Recognition;Roll / Spin and Move   
6986                                     Co-operative Play;Dice Rolling;Pick-up and Deliver   
6987                                                                                 Memory   
6988                                                                   Roll / Spin and Move   
6989                                                             Hex-and-Counter;Simulation   
6990                                                                           Dice Rolling   

                                                                                          type  \
0                                                            Admin: Better Description Needed!   
1                                   007 James Bond Action Episode Games;Characters: James Bond   
2                                                                                          NaN   
3                                                                                          NaN   
4                                   007 James Bond Action Episode Games;Characters: James Bond   
5                                                                                          NaN   
6                                                            Admin: Better Description Needed!   
7                                                                                          NaN   
8                                                            Animals: Lizards;Animals: Spiders   
9                                                                       3D Games;Animals: Dogs   
10                                                                                         NaN   
11                                                                                     Witches   
12                                                                                         NaN   
13                                                                                  Tube Games   
14                                                                                 Polyominoes   
15                                                                                         NaN   
16                                                                                         NaN   
17                                                                   Sports: Football / Soccer   
18                                                                                         NaN   
19                                                    Country: Korea;Operational Combat Series   
20                                                                                         NaN   
21                                                                            B-Movies;Samurai   
22                                                                             Aliens;B-Movies   
23                                                                                         NaN   
24                                                                                         NaN   
25                                                                                         NaN   
26                                                                                         NaN   
27                                                                 Post Office Stamps and Mail   
28                                                                                         NaN   
29                                                           Admin: Better Description Needed!   
...                                                                                        ...   
6961                                                                                       NaN   
6962                                                                                       NaN   
6963                                                                                       NaN   
6964  CTGs (Collectible Token Games);From RPG books to board games;The Weird West (BGG Family)   
6965                                                                                       NaN   
6966                                                                                       NaN   
6967                                                                                      Mars   
6968                                                                                       NaN   
6969                                                             CCGs (Collectible Card Games)   
6970                                                                                       NaN   
6971                                                                              Sports: Golf   
6972                                                                                       NaN   
6973                                                                                       NaN   
6974                                                                                       NaN   
6975                     Anime & Manga;CCGs (Collectible Card Games);Sports: Football / Soccer   
6976                                                                           Abacus wood box   
6977                                                                                       NaN   
6978                                                                                       NaN   
6979                                                                               Time Travel   
6980                                             Murder à la carte;Murder Mystery Party Series   
6981                                                          Food / Cooking;Murder à la carte   
6982                                                                               Chess Games   
6983                                                                         Murder à la carte   
6984                                                               Murder Mystery Party Series   
6985                                                                                       NaN   
6986                                                             Animals: Hedgehogs/Porcupines   
6987                                                 Animals: Fishes;Sports: Fishing / Angling   
6988                                                                                       NaN   
6989                                                                                       NaN   
6990                                                                                       NaN   

     BGG_link store_names  store_prices  links_to_buy  availability  \
0         NaN         NaN           NaN           NaN           NaN   
1         NaN         NaN           NaN           NaN           NaN   
2         NaN         NaN           NaN           NaN           NaN   
3         NaN         NaN           NaN           NaN           NaN   
4         NaN         NaN           NaN           NaN           NaN   
5         NaN         NaN           NaN           NaN           NaN   
6         NaN         NaN           NaN           NaN           NaN   
7         NaN         NaN           NaN           NaN           NaN   
8         NaN         NaN           NaN           NaN           NaN   
9         NaN         NaN           NaN           NaN           NaN   
10        NaN         NaN           NaN           NaN           NaN   
11        NaN         NaN           NaN           NaN           NaN   
12        NaN         NaN           NaN           NaN           NaN   
13        NaN         NaN           NaN           NaN           NaN   
14        NaN         NaN           NaN           NaN           NaN   
15        NaN         NaN           NaN           NaN           NaN   
16        NaN         NaN           NaN           NaN           NaN   
17        NaN         NaN           NaN           NaN           NaN   
18        NaN         NaN           NaN           NaN           NaN   
19        NaN         NaN           NaN           NaN           NaN   
20        NaN         NaN           NaN           NaN           NaN   
21        NaN         NaN           NaN           NaN           NaN   
22        NaN         NaN           NaN           NaN           NaN   
23        NaN         NaN           NaN           NaN           NaN   
24        NaN         NaN           NaN           NaN           NaN   
25        NaN         NaN           NaN           NaN           NaN   
26        NaN         NaN           NaN           NaN           NaN   
27        NaN         NaN           NaN           NaN           NaN   
28        NaN         NaN           NaN           NaN           NaN   
29        NaN         NaN           NaN           NaN           NaN   
...       ...         ...           ...           ...           ...   
6961      NaN         NaN           NaN           NaN           NaN   
6962      NaN         NaN           NaN           NaN           NaN   
6963      NaN         NaN           NaN           NaN           NaN   
6964      NaN         NaN           NaN           NaN           NaN   
6965      NaN         NaN           NaN           NaN           NaN   
6966      NaN         NaN           NaN           NaN           NaN   
6967      NaN         NaN           NaN           NaN           NaN   
6968      NaN         NaN           NaN           NaN           NaN   
6969      NaN         NaN           NaN           NaN           NaN   
6970      NaN         NaN           NaN           NaN           NaN   
6971      NaN         NaN           NaN           NaN           NaN   
6972      NaN         NaN           NaN           NaN           NaN   
6973      NaN         NaN           NaN           NaN           NaN   
6974      NaN         NaN           NaN           NaN           NaN   
6975      NaN         NaN           NaN           NaN           NaN   
6976      NaN         NaN           NaN           NaN           NaN   
6977      NaN         NaN           NaN           NaN           NaN   
6978      NaN         NaN           NaN           NaN           NaN   
6979      NaN         NaN           NaN           NaN           NaN   
6980      NaN         NaN           NaN           NaN           NaN   
6981      NaN         NaN           NaN           NaN           NaN   
6982      NaN         NaN           NaN           NaN           NaN   
6983      NaN         NaN           NaN           NaN           NaN   
6984      NaN         NaN           NaN           NaN           NaN   
6985      NaN         NaN           NaN           NaN           NaN   
6986      NaN         NaN           NaN           NaN           NaN   
6987      NaN         NaN           NaN           NaN           NaN   
6988      NaN         NaN           NaN           NaN           NaN   
6989      NaN         NaN           NaN           NaN           NaN   
6990      NaN         NaN           NaN           NaN           NaN   

      international_store  
0                     NaN  
1                     NaN  
2                     NaN  
3                     NaN  
4                     NaN  
5                     NaN  
6                     NaN  
7                     NaN  
8                     NaN  
9                     NaN  
10                    NaN  
11                    NaN  
12                    NaN  
13                    NaN  
14                    NaN  
15                    NaN  
16                    NaN  
17                    NaN  
18                    NaN  
19                    NaN  
20                    NaN  
21                    NaN  
22                    NaN  
23                    NaN  
24                    NaN  
25                    NaN  
26                    NaN  
27                    NaN  
28                    NaN  
29                    NaN  
...                   ...  
6961                  NaN  
6962                  NaN  
6963                  NaN  
6964                  NaN  
6965                  NaN  
6966                  NaN  
6967                  NaN  
6968                  NaN  
6969                  NaN  
6970                  NaN  
6971                  NaN  
6972                  NaN  
6973                  NaN  
6974                  NaN  
6975                  NaN  
6976                  NaN  
6977                  NaN  
6978                  NaN  
6979                  NaN  
6980                  NaN  
6981                  NaN  
6982                  NaN  
6983                  NaN  
6984                  NaN  
6985                  NaN  
6986                  NaN  
6987                  NaN  
6988                  NaN  
6989                  NaN  
6990                  NaN  

[6991 rows x 22 columns], 'corres': [('id', 'id'), ('name', 'name'), ('year', 'year'), ('rating', 'rating'), ('rank', 'rank'), ('num_players', 'num_players'), ('min_num_players', 'min_num_players'), ('max_num_players', 'max_num_players'), ('gameplay_time', 'gameplay_time'), ('min_gameplay_time', 'min_gameplay_time'), ('max_gameplay_time', 'max_gameplay_time'), ('min_age', 'min_age'), ('complexity_weight', 'complexity_weight'), ('category', 'category'), ('mechanisms', 'mechanisms'), ('type', 'type'), ('BGG_link', 'BGG_link'), ('store_names', 'store_names'), ('store_prices', 'store_prices'), ('links_to_buy', 'links_to_buy'), ('availability', 'availability'), ('international_store', 'international_store')]}

Generating Features

Here, we generate all the features we decided upon after our final iteration of cross validation and debugging. We only use the relevant subset of all these features in the reported iterations below.


In [11]:
# Generate a set of features
#import pdb; pdb.set_trace();
import py_entitymatching.feature.attributeutils as au
import py_entitymatching.feature.simfunctions as sim
import py_entitymatching.feature.tokenizers as tok

ltable = A1
rtable = B1

# Get similarity functions for generating the features for matching
sim_funcs = sim.get_sim_funs_for_matching()
# Get tokenizer functions for generating the features for matching
tok_funcs = tok.get_tokenizers_for_matching()

# Get the attribute types of the input tables
attr_types_ltable = au.get_attr_types(ltable)
attr_types_rtable = au.get_attr_types(rtable)

# Get the attribute correspondence between the input tables
attr_corres = au.get_attr_corres(ltable, rtable)
print(attr_types_ltable['name'])
print(attr_types_rtable['name'])
attr_types_ltable['name'] = 'str_bt_5w_10w'
attr_types_rtable['name'] = 'str_bt_5w_10w'



# Get the features
F = em.get_features(ltable, rtable, attr_types_ltable,
                                 attr_types_rtable, attr_corres,
                                 tok_funcs, sim_funcs)

#F = em.get_features_for_matching(A1, B1)
print(F['feature_name'])


#TODO get name feature!
#http://pradap-www.cs.wisc.edu/cs638/py_entitymatching/user-manual/_modules/py_entitymatching/feature/simfunctions.html#get_sim_funs_for_matching
#name_feature = em.get_feature_fn('name', em.get_tokenizers_for_matching(), em.get_sim_funs_for_matching())
#print(name_feature)
#em.add_feature(F, 'name_dist', name_feature)
#print(F['feature_name'])


Column BGG_link does not seem to qualify as any atomic type. It may contain all NaNs. Currently, setting its type to be un_determined.We recommend the users to manually update the returned types or features later. 

Column store_names does not seem to qualify as any atomic type. It may contain all NaNs. Currently, setting its type to be un_determined.We recommend the users to manually update the returned types or features later. 

Column store_prices does not seem to qualify as any atomic type. It may contain all NaNs. Currently, setting its type to be un_determined.We recommend the users to manually update the returned types or features later. 

Column links_to_buy does not seem to qualify as any atomic type. It may contain all NaNs. Currently, setting its type to be un_determined.We recommend the users to manually update the returned types or features later. 

Column availability does not seem to qualify as any atomic type. It may contain all NaNs. Currently, setting its type to be un_determined.We recommend the users to manually update the returned types or features later. 

Column international_store does not seem to qualify as any atomic type. It may contain all NaNs. Currently, setting its type to be un_determined.We recommend the users to manually update the returned types or features later. 

Column rating does not seem to qualify as any atomic type. It may contain all NaNs. Currently, setting its type to be un_determined.We recommend the users to manually update the returned types or features later. 

Column rank does not seem to qualify as any atomic type. It may contain all NaNs. Currently, setting its type to be un_determined.We recommend the users to manually update the returned types or features later. 

Column complexity_weight does not seem to qualify as any atomic type. It may contain all NaNs. Currently, setting its type to be un_determined.We recommend the users to manually update the returned types or features later. 

Column category does not seem to qualify as any atomic type. It may contain all NaNs. Currently, setting its type to be un_determined.We recommend the users to manually update the returned types or features later. 

Column mechanisms does not seem to qualify as any atomic type. It may contain all NaNs. Currently, setting its type to be un_determined.We recommend the users to manually update the returned types or features later. 

Column type does not seem to qualify as any atomic type. It may contain all NaNs. Currently, setting its type to be un_determined.We recommend the users to manually update the returned types or features later. 

str_bt_1w_5w
str_bt_5w_10w
0                                           id_id_exm
1                                           id_id_anm
2                                      id_id_lev_dist
3                                       id_id_lev_sim
4                           name_name_jac_qgm_3_qgm_3
5                       name_name_cos_dlm_dc0_dlm_dc0
6                                       name_name_mel
7                                  name_name_lev_dist
8                                   name_name_lev_sim
9                                       year_year_exm
10                                      year_year_anm
11                                 year_year_lev_dist
12                                  year_year_lev_sim
13                   num_players_num_players_lev_dist
14                    num_players_num_players_lev_sim
15                        num_players_num_players_jar
16                        num_players_num_players_jwn
17                        num_players_num_players_exm
18            num_players_num_players_jac_qgm_3_qgm_3
19                min_num_players_min_num_players_exm
20                min_num_players_min_num_players_anm
21           min_num_players_min_num_players_lev_dist
22            min_num_players_min_num_players_lev_sim
23                max_num_players_max_num_players_exm
24                max_num_players_max_num_players_anm
25           max_num_players_max_num_players_lev_dist
26            max_num_players_max_num_players_lev_sim
27        gameplay_time_gameplay_time_jac_qgm_3_qgm_3
28    gameplay_time_gameplay_time_cos_dlm_dc0_dlm_dc0
29    gameplay_time_gameplay_time_jac_dlm_dc0_dlm_dc0
30                    gameplay_time_gameplay_time_mel
31               gameplay_time_gameplay_time_lev_dist
32                gameplay_time_gameplay_time_lev_sim
33                    gameplay_time_gameplay_time_nmw
34                     gameplay_time_gameplay_time_sw
35            min_gameplay_time_min_gameplay_time_exm
36            min_gameplay_time_min_gameplay_time_anm
37       min_gameplay_time_min_gameplay_time_lev_dist
38        min_gameplay_time_min_gameplay_time_lev_sim
39            max_gameplay_time_max_gameplay_time_exm
40            max_gameplay_time_max_gameplay_time_anm
41       max_gameplay_time_max_gameplay_time_lev_dist
42        max_gameplay_time_max_gameplay_time_lev_sim
Name: feature_name, dtype: object

Cross Validation Method


In [12]:
def cross_validation_eval(H):
    cv_iter = pd.DataFrame(columns=['Precision', 'Recall', 'F1'])

    # Matchers
    matchers = [em.DTMatcher(name='DecisionTree', random_state=0),
     em.RFMatcher(name='RandomForest', random_state=0),
     em.SVMMatcher(name='SVM', random_state=0),
     em.NBMatcher(name='NaiveBayes'),
     em.LogRegMatcher(name='LogReg', random_state=0),
    ]
    
    for m in matchers:
        prec_result = em.select_matcher([m], table=H, 
                exclude_attrs=['_id', 'ltable_id', 'rtable_id','label'],
                k=5,
                target_attr='label', metric='precision', random_state=0)
        recall_result = em.select_matcher([m], table=H, 
                exclude_attrs=['_id', 'ltable_id', 'rtable_id','label'],
                k=5,
                target_attr='label', metric='recall', random_state=0)
        f1_result = em.select_matcher([m], table=H, 
                exclude_attrs=['_id', 'ltable_id', 'rtable_id','label'],
                k=5,
                target_attr='label', metric='f1', random_state=0)
        cv_iter = cv_iter.append(
            pd.DataFrame([
                [prec_result['cv_stats']['Mean score'][0],
                recall_result['cv_stats']['Mean score'][0],
                f1_result['cv_stats']['Mean score'][0],
                ]],
                index=[m.name],
                columns=['Precision', 'Recall', 'F1']))
    return cv_iter

Iteration 1: CV


In [13]:
# Subset of features we used on our first iteration
include_features = [
    'min_num_players_min_num_players_lev_dist',
    'max_num_players_max_num_players_lev_dist',
    'min_gameplay_time_min_gameplay_time_lev_dist',
    'max_gameplay_time_max_gameplay_time_lev_dist',
]
F_1 = F.loc[F['feature_name'].isin(include_features)]

In [14]:
# Convert the I into a set of feature vectors using F
H_1 = em.extract_feature_vecs(I, feature_table=F_1, attrs_after='label', show_progress=False)
H_1.head(10)


Out[14]:
_id ltable_id rtable_id min_num_players_min_num_players_lev_dist max_num_players_max_num_players_lev_dist min_gameplay_time_min_gameplay_time_lev_dist max_gameplay_time_max_gameplay_time_lev_dist label
153 65346 4081 4137 0.0 0.0 1.0 1.0 0
114 71437 4465 1000 0.0 0.0 0.0 0.0 1
322 98156 4574 4319 0.0 0.0 0.0 0.0 1
128 75832 5717 7538 0.0 0.0 0.0 0.0 1
105 165132 466 4483 0.0 0.0 0.0 0.0 1
95 212333 655 3448 0.0 1.0 2.0 2.0 0
8 44210 3395 811 0.0 1.0 1.0 1.0 0
56 199746 5204 10242 0.0 0.0 0.0 0.0 1
260 120176 2998 4173 0.0 0.0 2.0 2.0 0
333 46418 5900 811 0.0 1.0 2.0 2.0 0

In [15]:
cross_validation_eval(H_1)


Out[15]:
Precision Recall F1
DecisionTree 0.883431 0.965278 0.920400
RandomForest 0.883431 0.965278 0.920400
SVM 0.866166 0.965278 0.910821
NaiveBayes 0.702381 0.976389 0.814279
LogReg 0.882955 0.955278 0.915278

Iteration 2: Debug


In [16]:
PQ = em.split_train_test(H_1, train_proportion=0.80, random_state=0)
P = PQ['train']
Q = PQ['test']

In [17]:
# Convert the I into a set of feature vectors using F
# Here, we add name edit distance as a feature
include_features_2 = [
    'min_num_players_min_num_players_lev_dist',
    'max_num_players_max_num_players_lev_dist',
    'min_gameplay_time_min_gameplay_time_lev_dist',
    'max_gameplay_time_max_gameplay_time_lev_dist',
    'name_name_lev_dist'
]
F_2 = F.loc[F['feature_name'].isin(include_features_2)]
H_2 = em.extract_feature_vecs(I, feature_table=F_2, attrs_after='label', show_progress=False)
H_2.head(10)
# Split H into P and Q
PQ = em.split_train_test(H_2, train_proportion=0.75, random_state=0)
P = PQ['train']
Q = PQ['test']

Iteration 3: CV


In [18]:
# Convert the I into a set of feature vectors using F
# Here, we add name edit distance as a feature
include_features_3 = [
    'min_num_players_min_num_players_lev_dist',
    'max_num_players_max_num_players_lev_dist',
    'min_gameplay_time_min_gameplay_time_lev_dist',
    'max_gameplay_time_max_gameplay_time_lev_dist',
    'name_name_lev_dist'
]
F_3 = F.loc[F['feature_name'].isin(include_features_3)]
H_3 = em.extract_feature_vecs(I, feature_table=F_3, attrs_after='label', show_progress=False)

In [19]:
cross_validation_eval(H_3)


Out[19]:
Precision Recall F1
DecisionTree 0.882392 0.920359 0.898782
RandomForest 0.910866 0.931013 0.919163
SVM 0.937895 0.898137 0.916329
NaiveBayes 0.821319 0.964624 0.882312
LogReg 0.942857 0.943513 0.941584

In [ ]:

Iteration 4: CV


In [20]:
# Convert the I into a set of feature vectors using F
# Here, we add name edit distance as a feature
include_features_4 = [
    'min_num_players_min_num_players_lev_dist',
    'max_num_players_max_num_players_lev_dist',
    'min_gameplay_time_min_gameplay_time_lev_dist',
    'max_gameplay_time_max_gameplay_time_lev_dist',
    'name_name_jac_qgm_3_qgm_3'
]
F_4 = F.loc[F['feature_name'].isin(include_features_4)]
H_4 = em.extract_feature_vecs(I, feature_table=F_4, attrs_after='label', show_progress=False)
cross_validation_eval(H_4)


Out[20]:
Precision Recall F1
DecisionTree 0.907179 0.953595 0.926037
RandomForest 0.925278 0.921748 0.920029
SVM 0.898302 0.964624 0.926241
NaiveBayes 0.783056 0.976389 0.861230
LogReg 0.917749 0.953513 0.933086

Train-Test Set Accuracy


In [22]:
# Apply train, test set evaluation
I_table = em.extract_feature_vecs(I, feature_table=F_2, attrs_after='label', show_progress=False)
J_table = em.extract_feature_vecs(J, feature_table=F_2, attrs_after='label', show_progress=False)

matchers = [
 #em.DTMatcher(name='DecisionTree', random_state=0),
 #em.RFMatcher(name='RF', random_state=0),
 #em.NBMatcher(name='NaiveBayes'),
 em.LogRegMatcher(name='LogReg', random_state=0),
 #em.SVMMatcher(name='SVM', random_state=0)
]

for m in matchers:
    m.fit(table=I_table, exclude_attrs=['_id', 'ltable_id', 'rtable_id','label'], target_attr='label')
    J_table['prediction'] = m.predict(
        table=J_table, 
        exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'label'], 
        target_attr='label',
    )
    print(m.name)
    em.print_eval_summary(em.eval_matches(J_table, 'label', 'prediction'))
    J_table.drop('prediction', axis=1, inplace=True)
    print('')
    
log_matcher = matchers[0]


LogReg
Precision : 94.29% (33/35)
Recall : 97.06% (33/34)
F1 : 95.65%
False positives : 2 (out of 35 positive predictions)
False negatives : 1 (out of 65 negative predictions)


In [23]:
J_table['prediction'] = m.predict(
        table=J_table, 
        exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'label'], 
        target_attr='label',
    )
print(m.name)
em.print_eval_summary(em.eval_matches(J_table, 'label', 'prediction'))
J_table.drop('prediction', axis=1, inplace=True)
print('')
candidate_set_C1.csv


LogReg
Precision : 94.29% (33/35)
Recall : 97.06% (33/34)
F1 : 95.65%
False positives : 2 (out of 35 positive predictions)
False negatives : 1 (out of 65 negative predictions)