The objective of this assignment is to learn about simple data curation practices, and familiarize you with some of the data we'll be reusing later.
This notebook uses the notMNIST dataset to be used with python experiments. This dataset is designed to look like the classic MNIST dataset, while looking a little more like real data: it's a harder task, and the data is a lot less 'clean' than MNIST.
In [1]:
# These are all the modules we'll be using later. Make sure you can import them
# before proceeding further.
from __future__ import print_function
import matplotlib.pyplot as plt
import numpy as np
import os
import sys
import time
from datetime import timedelta
import tarfile
from IPython.display import display, Image
from scipy import ndimage
from scipy.spatial import distance
from sklearn.linear_model import LogisticRegression
from six.moves.urllib.request import urlretrieve
from six.moves import cPickle as pickle
from ipyparallel import Client, require
# Config the matplotlib backend as plotting inline in IPython
%matplotlib inline
In [15]:
%run label_util.py
In [6]:
def draw_images(label, a_arr, b_arr, bins_size=20):
x = np.array(range(bins_size))
f, (ax1, ax2, ax3, ax4) = plt.subplots(1, 4, figsize=(8, 1))
h_a = np.histogram(a_arr, bins=bins_size)
h_b = np.histogram(b_arr, bins=bins_size)
ax1.imshow(a_arr)
ax1.set_title('Label: ' + label)
ax2.bar(x, h_a[0])
ax3.imshow(b_arr)
ax4.bar(x, h_b[0])
plt.show()
def overlapping_comparison(counters, dataset_a, dataset_b, number_shown = 3):
letters = list(counters.keys())
i = 0
shown = 0
while shown < number_shown and i < len(letters):
character = letters[i]
similar_keys = list(counters[character].keys())
key = similar_keys[0]
if key == 'counter' and len(similar_keys) > 1:
key = similar_keys[1]
idx_a = int(key)
idx_b = counters[character][key][0]
label = '{0} (Distance: {1}) [{2}] - [{3}]'.format(character, manhattan_distance(dataset_a[idx_a], dataset_b[idx_b]), idx_a, idx_b)
draw_images(label, dataset_a[idx_a], dataset_b[idx_b])
i += 1
shown += 1
def display_overlap(counters):
key_lst = sorted(counters.keys())
total = 0
for key in key_lst:
total += counters[key]['counter']
print('Label {0}: {1}'.format(key, counters[key]['counter']))
print('Total:', total)
def wrap_tuples(labels, dataset):
result = []
for idx, item in enumerate(zip(labels, dataset)):
result.append((idx, item[0], item[1]))
return result
def is_equal_comparison(a_arr, b_arr):
return (a_arr==b_arr).all()
def euclidean_distance(a_arr, b_arr):
'''Euclidean distance without the sqrt'''
return np.sum(np.power(a_arr - b_arr, 2))
@require('numpy as np')
def manhattan_distance(a_arr, b_arr):
return np.sum(np.absolute(a_arr - b_arr))
def count_duplication(counters, lbl, idxA, idxB):
str_lbl = get_char_by_lbl(lbl)
if str_lbl not in counters:
counters[str_lbl] = {}
counters[str_lbl]['counter'] = 0
counters[str_lbl]['counter'] += 1
if str(idxA) not in counters[str_lbl]:
counters[str_lbl][str(idxA)] = []
counters[str_lbl][str(idxA)].append(idxB)
def count_equal_data(label_lst_A, data_lst_A, label_lst_B, data_lst_B, distance_threshold=0, min_distance_threshold = 0):
start_time = time.clock()
counters = {}
for idxA, lblA in enumerate(label_lst_A):
for idxB, lblB in enumerate(label_lst_B):
if lblA == lblB:
itemA = data_lst_A[idxA]
itemB = data_lst_B[idxB]
if distance_threshold == 0 and is_equal_comparison(itemA, itemB):
count_duplication(counters, lblA, idxA, idxB)
if distance_threshold > 0 and distance_threshold >= manhattan_distance(itemA, itemB) > min_distance_threshold:
count_duplication(counters, lblA, idxA, idxB)
end_time = time.clock()
return (counters, timedelta(seconds=end_time - start_time))
def count_equal_tuples(tuple_lst_A, tuple_lst_B, distance_threshold=0, min_distance_threshold = 0):
idx_idx = 0
lbl_idx = 1
data_idx = 2
counters = {}
for item_A in tuple_lst_A:
for item_B in tuple_lst_B:
if item_A[lbl_idx] == item_B[lbl_idx]:
if distance_threshold == 0 and is_equal_comparison(item_A[data_idx], item_B[data_idx]):
count_duplication(counters, item_A[lbl_idx], item_A[idx_idx], item_B[idx_idx])
if distance_threshold > 0 and distance_threshold >= manhattan_distance(item_A[data_idx], item_B[data_idx]) > min_distance_threshold:
count_duplication(counters, item_A[lbl_idx], item_A[idx_idx], item_B[idx_idx])
return counters
@require(get_char_by_lbl)
def count_duplication(counters, lbl, idxA, idxB):
str_lbl = get_char_by_lbl(lbl)
if str_lbl not in counters:
counters[str_lbl] = {}
counters[str_lbl]['counter'] = 0
counters[str_lbl]['counter'] += 1
if str(idxA) not in counters[str_lbl]:
counters[str_lbl][str(idxA)] = []
counters[str_lbl][str(idxA)].append(idxB)
@require(is_equal_comparison, count_duplication, manhattan_distance)
def item_acync_handler():
idx_idx = 0
lbl_idx = 1
data_idx = 2
for item_A in tuple_lst_A:
for item_B in tuple_lst_B:
if item_A[lbl_idx] == item_B[lbl_idx]:
if distance_threshold == 0 and is_equal_comparison(item_A[data_idx], item_B[data_idx]):
count_duplication(counters, item_A[lbl_idx], item_A[idx_idx], item_B[idx_idx])
if distance_threshold > 0 and distance_threshold >= manhattan_distance(item_A[data_idx], item_B[data_idx]) > min_distance_threshold:
count_duplication(counters, item_A[lbl_idx], item_A[idx_idx], item_B[idx_idx])
def reduce_counters(counters_lst):
result = {}
for counters in counters_lst:
for letter_key, item in counters.items():
if letter_key not in result:
result[letter_key] = {'counter': 0}
for key, value in item.items():
if key == 'counter':
result[letter_key][key] += value
elif key not in result[letter_key]:
result[letter_key][key] = value
else:
for idx in value:
result[letter_key][key].append(idx)
return result
def count_equal_tuples_parallel(tuple_lst_A, tuple_lst_B, distance_threshold=0, min_distance_threshold = 0):
rc = Client()
dview = rc[:]
dview.push(dict(tuple_lst_B = tuple_lst_B, map_dict=map_dict,
distance_threshold=distance_threshold,
min_distance_threshold=min_distance_threshold))
dview['counters'] = {}
dview.scatter('tuple_lst_A', tuple_lst_A)
dview.block=True
dview.apply(item_acync_handler)
result = reduce_counters(dview['counters'])
return result
First, we'll download the dataset to our local machine. The data consists of characters rendered in a variety of fonts on a 28x28 image. The labels are limited to 'A' through 'J' (10 classes). The training set has about 500k and the testset 19000 labelled examples. Given these sizes, it should be possible to train models quickly on any machine.
In [7]:
url = 'http://commondatastorage.googleapis.com/books1000/'
last_percent_reported = None
def download_progress_hook(count, blockSize, totalSize):
"""A hook to report the progress of a download. This is mostly intended for users with
slow internet connections. Reports every 1% change in download progress.
"""
global last_percent_reported
percent = int(count * blockSize * 100 / totalSize)
if last_percent_reported != percent:
if percent % 5 == 0:
sys.stdout.write("%s%%" % percent)
sys.stdout.flush()
else:
sys.stdout.write(".")
sys.stdout.flush()
last_percent_reported = percent
def maybe_download(filename, expected_bytes, force=False):
"""Download a file if not present, and make sure it's the right size."""
if force or not os.path.exists(filename):
print('Attempting to download:', filename)
filename, _ = urlretrieve(url + filename, filename, reporthook=download_progress_hook)
print('\nDownload Complete!')
statinfo = os.stat(filename)
if statinfo.st_size == expected_bytes:
print('Found and verified', filename)
else:
raise Exception(
'Failed to verify ' + filename + '. Can you get to it with a browser?')
return filename
train_filename = maybe_download('notMNIST_large.tar.gz', 247336696)
test_filename = maybe_download('notMNIST_small.tar.gz', 8458043)
Extract the dataset from the compressed .tar.gz file. This should give you a set of directories, labelled A through J.
In [8]:
num_classes = 10
np.random.seed(133)
def maybe_extract(filename, force=False):
root = os.path.splitext(os.path.splitext(filename)[0])[0] # remove .tar.gz
if os.path.isdir(root) and not force:
# You may override by setting force=True.
print('%s already present - Skipping extraction of %s.' % (root, filename))
else:
print('Extracting data for %s. This may take a while. Please wait.' % root)
tar = tarfile.open(filename)
sys.stdout.flush()
tar.extractall()
tar.close()
data_folders = [
os.path.join(root, d) for d in sorted(os.listdir(root))
if os.path.isdir(os.path.join(root, d))]
if len(data_folders) != num_classes:
raise Exception(
'Expected %d folders, one per class. Found %d instead.' % (
num_classes, len(data_folders)))
print(data_folders)
return data_folders
train_folders = maybe_extract(train_filename)
test_folders = maybe_extract(test_filename)
In [9]:
from IPython.display import Image, display
num_first_items = 1
def display_first_items(folder_path):
print('Letter:', folder_path[-1:])
lst = os.listdir(folder_path)[:num_first_items]
for file_name in lst:
full_file_name = os.path.join(folder_path, file_name)
display(Image(filename=full_file_name))
for folder in train_folders:
display_first_items(folder)
for folder in test_folders:
display_first_items(folder)
Now let's load the data in a more manageable format. Since, depending on your computer setup you might not be able to fit it all in memory, we'll load each class into a separate dataset, store them on disk and curate them independently. Later we'll merge them into a single dataset of manageable size.
We'll convert the entire dataset into a 3D array (image index, x, y) of floating point values, normalized to have approximately zero mean and standard deviation ~0.5 to make training easier down the road.
A few images might not be readable, we'll just skip them.
In [10]:
image_size = 28 # Pixel width and height.
pixel_depth = 255.0 # Number of levels per pixel.
def load_letter(folder, min_num_images):
"""Load the data for a single letter label."""
image_files = os.listdir(folder)
dataset = np.ndarray(shape=(len(image_files), image_size, image_size),
dtype=np.float32)
print(folder)
num_images = 0
for image in image_files:
image_file = os.path.join(folder, image)
try:
image_data = (ndimage.imread(image_file).astype(float) -
pixel_depth / 2) / pixel_depth
if image_data.shape != (image_size, image_size):
raise Exception('Unexpected image shape: %s' % str(image_data.shape))
if image_data.mean() == 0.5:
print('No data in image:', image_file)
continue
dataset[num_images, :, :] = image_data
num_images = num_images + 1
except IOError as e:
print('Could not read:', image_file, ':', e, '- it\'s ok, skipping.')
dataset = dataset[0:num_images, :, :]
if num_images < min_num_images:
raise Exception('Many fewer images than expected: %d < %d' %
(num_images, min_num_images))
print('Full dataset tensor:', dataset.shape)
print('Mean:', np.mean(dataset))
print('Standard deviation:', np.std(dataset))
return dataset
def maybe_pickle(data_folders, min_num_images_per_class, force=False):
dataset_names = []
for folder in data_folders:
set_filename = folder + '.pickle'
dataset_names.append(set_filename)
if os.path.exists(set_filename) and not force:
# You may override by setting force=True.
print('%s already present - Skipping pickling.' % set_filename)
else:
print('Pickling %s.' % set_filename)
dataset = load_letter(folder, min_num_images_per_class)
try:
with open(set_filename, 'wb') as f:
pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)
except Exception as e:
print('Unable to save data to', set_filename, ':', e)
return dataset_names
train_datasets = maybe_pickle(train_folders, 45000)
test_datasets = maybe_pickle(test_folders, 1600)
In [11]:
def show_pickle(file_path):
print(file_path)
with open(file_path, 'rb') as f:
dataset = pickle.load(f)
plt.figure(figsize=(1,1))
plt.imshow(dataset[1])
plt.show()
for pickle_file in train_datasets:
show_pickle(pickle_file)
for pickle_file in test_datasets:
show_pickle(pickle_file)
In [12]:
def show_pickle_stats(file_path):
with open(file_path, 'rb') as f:
dataset = pickle.load(f)
print(file_path, len(dataset))
for pickle_file in train_datasets:
show_pickle_stats(pickle_file)
for pickle_file in test_datasets:
show_pickle_stats(pickle_file)
Merge and prune the training data as needed. Depending on your computer setup, you might not be able to fit it all in memory, and you can tune train_size as needed. The labels will be stored into a separate array of integers 0 through 9.
Also create a validation dataset for hyperparameter tuning.
In [13]:
def make_arrays(nb_rows, img_size):
if nb_rows:
dataset = np.ndarray((nb_rows, img_size, img_size), dtype=np.float32)
labels = np.ndarray(nb_rows, dtype=np.int32)
else:
dataset, labels = None, None
return dataset, labels
def merge_datasets(pickle_files, train_size, valid_size=0):
num_classes = len(pickle_files)
valid_dataset, valid_labels = make_arrays(valid_size, image_size)
train_dataset, train_labels = make_arrays(train_size, image_size)
vsize_per_class = valid_size // num_classes
tsize_per_class = train_size // num_classes
start_v, start_t = 0, 0
end_v, end_t = vsize_per_class, tsize_per_class
end_l = vsize_per_class+tsize_per_class
for label, pickle_file in enumerate(pickle_files):
try:
with open(pickle_file, 'rb') as f:
letter_set = pickle.load(f)
# let's shuffle the letters to have random validation and training set
np.random.shuffle(letter_set)
if valid_dataset is not None:
valid_letter = letter_set[:vsize_per_class, :, :]
valid_dataset[start_v:end_v, :, :] = valid_letter
valid_labels[start_v:end_v] = label
start_v += vsize_per_class
end_v += vsize_per_class
train_letter = letter_set[vsize_per_class:end_l, :, :]
train_dataset[start_t:end_t, :, :] = train_letter
train_labels[start_t:end_t] = label
start_t += tsize_per_class
end_t += tsize_per_class
except Exception as e:
print('Unable to process data from', pickle_file, ':', e)
raise
return valid_dataset, valid_labels, train_dataset, train_labels
train_size = 200000
valid_size = 10000
test_size = 10000
valid_dataset, valid_labels, train_dataset, train_labels = merge_datasets(
train_datasets, train_size, valid_size)
_, _, test_dataset, test_labels = merge_datasets(test_datasets, test_size)
print('Training:', train_dataset.shape, train_labels.shape)
print('Validation:', valid_dataset.shape, valid_labels.shape)
print('Testing:', test_dataset.shape, test_labels.shape)
In [20]:
print('train labels:', count_labels(train_labels))
print('valid labels:', count_labels(valid_labels))
print('test labels:', count_labels(test_labels))
Next, we'll randomize the data. It's important to have the labels well shuffled for the training and test distributions to match.
In [21]:
def randomize(dataset, labels):
permutation = np.random.permutation(labels.shape[0])
shuffled_dataset = dataset[permutation,:,:]
shuffled_labels = labels[permutation]
return shuffled_dataset, shuffled_labels
train_dataset, train_labels = randomize(train_dataset, train_labels)
test_dataset, test_labels = randomize(test_dataset, test_labels)
valid_dataset, valid_labels = randomize(valid_dataset, valid_labels)
In [22]:
print('train labels:', count_labels(train_labels))
print('valid labels:', count_labels(valid_labels))
print('test labels:', count_labels(test_labels))
In [23]:
def show_data(dataset, labels, size=3):
print('=============================================')
for lbl, img_arr in zip(labels[:size], dataset[:size]):
print(map_dict[str(lbl)])
plt.figure(figsize=(1,1))
plt.imshow(img_arr)
plt.show()
show_data(train_dataset, train_labels)
show_data(test_dataset, test_labels)
show_data(valid_dataset, valid_labels)
Finally, let's save the data for later reuse:
In [14]:
pickle_file = 'notMNIST.pickle'
try:
f = open(pickle_file, 'wb')
save = {
'train_dataset': train_dataset,
'train_labels': train_labels,
'valid_dataset': valid_dataset,
'valid_labels': valid_labels,
'test_dataset': test_dataset,
'test_labels': test_labels,
}
pickle.dump(save, f, pickle.HIGHEST_PROTOCOL)
f.close()
except Exception as e:
print('Unable to save data to', pickle_file, ':', e)
raise
In [15]:
statinfo = os.stat(pickle_file)
print('Compressed pickle size:', statinfo.st_size)
In [4]:
bins_size = 28 * 4
def calc_histogram(dataset, bins = bins_size):
start_time = time.clock()
hist_list = []
for item in dataset:
hist = np.histogram(item, bins=bins)
hist_list.append(hist[0])
end_time = time.clock()
return (hist_list, timedelta(seconds=end_time - start_time))
In [120]:
train_histogram, calc_duration = calc_histogram(train_dataset, bins_size)
print('Histograms for train dataset calculates in', calc_duration)
valid_histogram, calc_duration = calc_histogram(valid_dataset, bins_size)
print('Histograms for validation dataset calculates in', calc_duration)
test_histogram, calc_duration = calc_histogram(test_dataset, bins_size)
print('Histograms for test dataset calculates in', calc_duration)
In [121]:
# pickle_hist_file = 'notMNIST.hist.pickle'
# try:
# f = open(pickle_hist_file, 'wb')
# save = {
# 'train_histogram': train_histogram,
# 'valid_histogram': valid_histogram,
# 'test_histogram': test_histogram,
# }
# pickle.dump(save, f, pickle.HIGHEST_PROTOCOL)
# f.close()
# except Exception as e:
# print('Unable to save data to', pickle_hist_file, ':', e)
# raise
# statinfo = os.stat(pickle_hist_file)
# print('Compressed histograms pickle size:', statinfo.st_size)
By construction, this dataset might contain a lot of overlapping samples, including training data that's also contained in the validation and test set! Overlap between training and test can skew the results if you expect to use your model in an environment where there is never an overlap, but are actually ok if you expect to see training samples recur when you use it. Measure how much overlap there is between training, validation and test samples.
Optional questions:
In [24]:
pickle_file = 'notMNIST.pickle'
pickle_hist_file = 'notMNIST.hist.pickle'
try:
with open(pickle_file, 'rb') as f:
save = pickle.load(f)
train_dataset = save['train_dataset']
train_labels = save['train_labels']
valid_dataset = save['valid_dataset']
valid_labels = save['valid_labels']
test_dataset = save['test_dataset']
test_labels = save['test_labels']
print('Training:', train_dataset.shape, train_labels.shape)
print('Validation:', valid_dataset.shape, valid_labels.shape)
print('Testing:', test_dataset.shape, test_labels.shape)
except Exception as e:
print('Unable to load full dataset to', pickle_file, ':', e)
raise
# try:
# with open(pickle_hist_file, 'rb') as f:
# save = pickle.load(f)
# train_histogram = save['train_histogram']
# valid_histogram = save['valid_histogram']
# test_histogram = save['test_histogram']
# print('Training histogram:', len(train_histogram))
# print('Validation histogram:', len(valid_histogram))
# print('Testing histogram:', len(test_histogram))
# except Exception as e:
# print('Unable to load full dataset to', pickle_file, ':', e)
# raise
In [25]:
start_time = time.clock()
train_tuple_lst = wrap_tuples(train_labels, train_dataset)
valid_tuple_lst = wrap_tuples(valid_labels, valid_dataset)
test_tuple_lst = wrap_tuples(test_labels, test_dataset)
end_time = time.clock()
print('Labels and data sets to tuples time:', timedelta(seconds=end_time - start_time))
For distance measurement between images used Manhattan metric (reference)
In [23]:
distance_overlapping = 10
In [24]:
start_time = time.clock()
overlap_valid_test = count_equal_tuples(valid_tuple_lst, test_tuple_lst)
end_time = time.clock()
duration = timedelta(seconds=end_time - start_time)
print('Counting overlapping between validation and test datasets during', duration)
display_overlap(overlap_valid_test)
In [25]:
start_time = time.clock()
overlap_valid_test_near = count_equal_tuples(valid_tuple_lst, test_tuple_lst, distance_overlapping)
end_time = time.clock()
duration = timedelta(seconds=end_time - start_time)
print('Counting overlapping between validation and test datasets (with overlaping distance) during', duration)
display_overlap(overlap_valid_test_near)
In [26]:
start_time = time.clock()
overlap_valid_test = count_equal_tuples_parallel(valid_tuple_lst, test_tuple_lst)
end_time = time.clock()
duration = timedelta(seconds=end_time - start_time)
print('Counting overlapping between validation and test datasets during', duration)
display_overlap(overlap_valid_test)
In [75]:
overlapping_comparison(overlap_valid_test, valid_dataset, test_dataset)
In [27]:
start_time = time.clock()
overlap_valid_test_near = count_equal_tuples_parallel(valid_tuple_lst, test_tuple_lst, distance_overlapping)
end_time = time.clock()
duration = timedelta(seconds=end_time - start_time)
print('Counting overlapping between validation and test datasets (with overlaping distance) during', duration)
display_overlap(overlap_valid_test_near)
In [76]:
overlapping_comparison(overlap_valid_test_near, valid_dataset, test_dataset)
In [61]:
start_time = time.clock()
overlap_valid_test_far = count_equal_tuples_parallel(valid_tuple_lst, test_tuple_lst, 110, 100)
end_time = time.clock()
duration = timedelta(seconds=end_time - start_time)
print('Counting overlapping between validation and test datasets (with overlaping interval) during', duration)
display_overlap(overlap_valid_test_far)
In [77]:
overlapping_comparison(overlap_valid_test_far, valid_dataset, test_dataset)
In [28]:
start_time = time.clock()
overlap_train_valid = count_equal_tuples_parallel(train_tuple_lst, valid_tuple_lst)
end_time = time.clock()
duration = timedelta(seconds=end_time - start_time)
print('Counting overlapping between validation and test datasets during', duration)
display_overlap(overlap_train_valid)
In [78]:
overlapping_comparison(overlap_train_valid, train_dataset, valid_dataset)
In [29]:
start_time = time.clock()
overlap_train_valid_near = count_equal_tuples_parallel(train_tuple_lst, valid_tuple_lst, distance_overlapping)
end_time = time.clock()
duration = timedelta(seconds=end_time - start_time)
print('Counting overlapping between validation and test datasets (with overlaping distance) during', duration)
display_overlap(overlap_train_valid_near)
In [80]:
overlapping_comparison(overlap_train_valid_near, train_dataset, valid_dataset)
In [30]:
start_time = time.clock()
overlap_train_test = count_equal_tuples_parallel(train_tuple_lst, test_tuple_lst)
end_time = time.clock()
duration = timedelta(seconds=end_time - start_time)
print('Counting overlapping between validation and test datasets during', duration)
display_overlap(overlap_train_test)
In [81]:
overlapping_comparison(overlap_train_test, train_dataset, test_dataset)
In [31]:
start_time = time.clock()
overlap_train_test_near = count_equal_tuples_parallel(train_tuple_lst, test_tuple_lst, distance_overlapping)
end_time = time.clock()
duration = timedelta(seconds=end_time - start_time)
print('Counting overlapping between validation and test datasets (with overlaping distance) during', duration)
display_overlap(overlap_train_test_near)
In [82]:
overlapping_comparison(overlap_train_test_near, train_dataset, test_dataset)
In [36]:
%timeit is_equal_comparison(item_a, item_b)
In [37]:
%timeit manhattan_distance(valid_histogram[8], valid_histogram[9])
In [38]:
%timeit distance.cityblock(item_a.flatten(), item_b.flatten())
Let's get an idea of what an off-the-shelf classifier can give you on this data. It's always good to check that there is something to learn, and that it's a problem that is not so trivial that a canned solution solves it.
Train a simple model on this data using 50, 100, 1000 and 5000 training samples. Hint: you can use the LogisticRegression model from sklearn.linear_model.
Optional question: train an off-the-shelf model on all the data!
In [27]:
from sklearn.linear_model import LogisticRegression
In [ ]:
%run label_util.py
# print('train labels:', count_labels(train_labels))
# print('valid labels:', count_labels(valid_labels))
# print('test labels:', count_labels(test_labels))
# show_data(train_dataset, train_labels)
# show_data(test_dataset, test_labels)
# show_data(valid_dataset, valid_labels)
In [44]:
from collections import Counter
cnt = Counter(valid_labels)
keys = cnt.keys()
one_class_size = 50 // len(keys)
for key in keys:
class_indexes = np.where(valid_labels == key)[0][:one_class_size]
print(type(valid_labels[class_indexes]), valid_labels[class_indexes])
In [42]:
valid_labels.shape
Out[42]:
In [ ]:
logreg = linear_model.LogisticRegression()