Codealong of Radek Osmulski's notebook establishing a CIFAR-10 baseline with the Fastai ImageNet WideResNet22. For a Fastai CV research collaboration.
Wayne Nixalo –– 2018/6/1
In [1]:
%matplotlib inline
%reload_ext autoreload
%autoreload 2
In [2]:
from fastai.conv_learner import *
# fastai/imagenet-fast/cifar10/models/ repo
from imagenet_fast_cifar_models.wideresnet import wrn_22
from torchvision import transforms, datasets
# allows you to enable the inbuilt cudnn auto-tuner to find the
# best algorithm for your hardware. https://discuss.pytorch.org/t/what-does-torch-backends-cudnn-benchmark-do/5936/2
torch.backends.cudnn.benchmark = True
PATH = Path("data/cifar10")
In [3]:
classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')
stats = (np.array([ 0.4914 , 0.48216, 0.44653]), np.array([ 0.24703, 0.24349, 0.26159]))
In [4]:
## temp small dataset for fast iteration
# import cifar_utils
# PATH = cifar_utils.create_cifar_subset(PATH, copydirs=['train','test'], p=0.1)
PATH = Path("data/cifar10_tmp")
In [13]:
## Aside: making a small subset of the dataset for fast troubleshooting
## also bc idk how to marry pytorch dataloaders w/ csv's yet.
import cifar_utils
PATH = cifar_utils.create_cifar_subset(PATH, copydirs=['train','test'], p=0.1)
In [4]:
## Aside: checking the normalization transforms
tensor = T(np.ones((3,32,32)))
t1 = transforms.Normalize(stats[0], stats[1])(tensor)
t2 = transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))(tensor)
np.unique(np.isclose(t1, t2))
Out[4]:
We construct the data object manually from low level components in a way that can be used with the fastsai library.
In [5]:
def get_loaders(bs, num_workers):
traindir = str(PATH/'train')
valdir = str(PATH/'test')
tfms = [transforms.ToTensor(),
transforms.Normalize(*stats)]
#transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))]
aug_tfms = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
] + tfms)
train_dataset = datasets.ImageFolder(traindir, aug_tfms)
val_dataset = datasets.ImageFolder(valdir, transforms.Compose(tfms))
aug_dataset = datasets.ImageFolder(valdir, aug_tfms)
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=bs, shuffle=True, num_workers=num_workers, pin_memory=False)
val_loader = torch.utils.data.DataLoader(
val_dataset, batch_size=bs, shuffle=False, num_workers=num_workers, pin_memory=False)
aug_loader = torch.utils.data.DataLoader(
aug_dataset, batch_size=bs, shuffle=False, num_workers=num_workers, pin_memory=False)
## NOTE--didnt work: Want automated GPU/CPU handling so using fastai dataloaders
# train_loader = DataLoader(train_dataset, batch_size=bs, shuffle=True,
# num_workers=num_workers, pin_memory=True)
# val_loader = DataLoader(val_dataset, batch_size=bs, shuffle=False,
# num_workers=num_workers, pin_memory=True)
# aug_loader = DataLoader(aug_dataset, batch_size=bs, shuffle=False,
# num_workers=num_workers, pin_memory=True)
return train_loader, val_loader, aug_loader
This is very similar to how Fastai initializes its ModelData, except Fastai uses the Pytorch default pin_memory=False.
What is the disadvantage of using pin_memory? –
Pytorch docs:
by pinning your batch in cpu memory, data transfer to gpu can be much faster
Soumith:
pinned memory is page-locked memory. It's easy to shoot yourself in the foot if you enable it for everything because it can't be pre-empted. ... if you're seeing system freeze or swap being used a lot, disable it.
In [6]:
def get_data(bs, num_workers):
trn_dl, val_dl, aug_dl = get_loaders(bs, num_workers)
data = ModelData(PATH, trn_dl, val_dl)
data.aug_dl = aug_dl
data.sz = 32
return data
def get_learner(arch, bs):
learn = ConvLearner.from_model_data(arch.cuda(), get_data(bs, num_cpus()))
learn.crit = nn.CrossEntropyLoss()
learn.metrics = [accuracy]
return learn
def get_TTA_accuracy(learn):
preds, targs = learn.TTA()
# combining the predictions across augmented and non augmented inputs
preds = 0.6 * preds[0] + 0.4 * preds[1:].sum(0)
return accuracy_np(preds, targs)
My copy of Radek's reimplementation of the FastAI DAWN Bench submission in terms of archutecture and training parameters – from the imagenet-fast repo.
In [ ]:
# learner = get_learner(wrn_22(), 512)
# learner.lr_find(wds=1e-4)
# learner.sched.plot(n_skip_end=1)
In [35]:
learner = get_learner(wrn_22(), 16)
learner.lr_find(wds=1e-4)
learner.sched.plot(n_skip_end=1)
In [33]:
learner = get_learner(wrn_22(), 16)
learner.lr_find(wds=1e-4)
learner.sched.plot(n_skip_end=1)
In [32]:
learner = get_learner(wrn_22(), 16)
learner.lr_find(wds=1e-4)
learner.sched.plot(n_skip_end=1)
In [21]:
learner = get_learner(wrn_22(), 8)
learner.lr_find(wds=1e-4)
learner.sched.plot(n_skip_end=1)
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [8]:
learner = get_learner(wrn_22(), 8)
learner.lr_find(wds=1e-4)
learner.sched.plot(n_skip_end=1)
I've been having trouble getting the pytorch dataloaders above to play nice: for some reason they won't return cuda tensors, only cpu float tensors... but the model is waiting to run on data on the gpu...
I don't want to manually build fastai dataloaders either – since in that case they require a validation split and more low-level thinking that'll distract a lot from wider issues; so I'm using the automated fastai way.
In [18]:
bs=8; sz=32; fulldata=False
In [19]:
aug_tfms = [RandomFlip(), RandomCrop(32)] # hopefully this is the same as aug_tfms above
tfms = tfms_from_stats(stats, sz=32, aug_tfms=aug_tfms, pad=4)
if not fulldata:
# quick prototyping csv (10% dataset)
val_idxs = get_cv_idxs(n=pd.read_csv(PATH/'tmp.csv').count()[0])
model_data = ImageClassifierData.from_csv(
PATH, 'train', PATH/'tmp.csv', bs=bs, tfms=tfms, val_idxs=val_idxs)
else:
# full dataset
model_data = ImageClassifierData.from_paths(
PATH, bs=bs, tfms=tfms, trn_name='train',val_name='test')
In [34]:
learner = ConvLearner.from_model_data(wrn_22(), model_data)
learner.crit = nn.CrossEntropyLoss(); learner.metrics = [accuracy]
learner.lr_find(wds=1e-4)
learner.sched.plot(n_skip_end=1)
In [20]:
learner = ConvLearner.from_model_data(wrn_22(), model_data)
learner.crit = nn.CrossEntropyLoss(); learner.metrics = [accuracy]
learner.lr_find(wds=1e-4)
learner.sched.plot(n_skip_end=1)
In [ ]:
In [ ]:
In [16]:
learner = ConvLearner.from_model_data(wrn_22(), model_data)
learner.crit = nn.CrossEntropyLoss(); learner.metrics = [accuracy]
learner.lr_find(wds=1e-4)
learner.sched.plot(n_skip_end=1)
In [ ]:
In [ ]:
In [ ]:
In [6]:
DataLoader.__init__
Out[6]:
In [7]:
torch.utils.data.DataLoader.__init__
Out[7]:
In [42]:
def get_loaders(bs, num_workers):
traindir = str(PATH/'train')
valdir = str(PATH/'test')
tfms = [transforms.ToTensor(),
transforms.Normalize(stats[0], stats[1])]
#transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))]
aug_tfms = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
] + tfms)
train_dataset = datasets.ImageFolder(traindir, aug_tfms)
val_dataset = datasets.ImageFolder(valdir, transforms.Compose(tfms))
aug_dataset = datasets.ImageFolder(valdir, aug_tfms)
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=bs, shuffle=True, num_workers=num_workers, pin_memory=False)
val_loader = torch.utils.data.DataLoader(
val_dataset, batch_size=bs, shuffle=False, num_workers=num_workers, pin_memory=False)
aug_loader = torch.utils.data.DataLoader(
aug_dataset, batch_size=bs, shuffle=False, num_workers=num_workers, pin_memory=False)
## Want automated GPU/CPU handling so using fastai dataloaders
# train_loader = DataLoader(train_dataset, batch_size=bs, shuffle=True,
# num_workers=num_workers, pin_memory=True)
# val_loader = DataLoader(val_dataset, batch_size=bs, shuffle=False,
# num_workers=num_workers, pin_memory=True)
# aug_loader = DataLoader(aug_dataset, batch_size=bs, shuffle=False,
# num_workers=num_workers, pin_memory=True)
return train_loader, val_loader, aug_loader
In [ ]:
In [130]:
bs=64; sz=32; num_workers=1
traindir = str(PATH/'train')
valdir = str(PATH/'test')
tfms = [transforms.ToTensor(),
transforms.Normalize(stats[0], stats[1])]
#transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))]
aug_tfms = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
] + tfms)
train_dataset = datasets.ImageFolder(traindir, aug_tfms)
val_dataset = datasets.ImageFolder(valdir, transforms.Compose(tfms))
aug_dataset = datasets.ImageFolder(valdir, aug_tfms)
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=bs, shuffle=True, num_workers=num_workers, pin_memory=True)
val_loader = torch.utils.data.DataLoader(
val_dataset, batch_size=bs, shuffle=False, num_workers=num_workers, pin_memory=True)
aug_loader = torch.utils.data.DataLoader(
aug_dataset, batch_size=bs, shuffle=False, num_workers=num_workers, pin_memory=True)
In [131]:
train_loader.cuda()
In [ ]:
In [ ]:
In [132]:
x,y = next(iter(train_loader))
In [133]:
x[0].type()
Out[133]:
In [135]:
type(y[0])
Out[135]:
In [29]:
tfms = tfms_from_stats(stats, sz=32)
md = ImageClassifierData.from_csv(PATH, 'train', PATH/'tmp.csv', tfms=tfms)
In [30]:
next(iter(md.trn_dl))[0].type()
Out[30]:
In [33]:
md.trn_ds.
Out[33]:
In [34]:
train_dataset.
Out[34]:
In [ ]:
In [ ]:
In [13]:
train_loader,_,_ = get_loaders(bs=64, num_workers=1)
In [14]:
next(iter(train_loader))[0].type()
In [13]:
## Aside: making a small subset of the dataset for fast troubleshooting
## also bc idk how to marry pytorch dataloaders w/ csv's yet.
import cifar_utils
PATH = cifar_utils.create_cifar_subset(PATH, copydirs=['train','test'], p=0.1)
In [28]:
# df.head()
In [38]:
import cifar_utils
In [53]:
df = cifar_utils.generate_csv(PATH)
df.to_csv(PATH/'tmp.csv', index=False)
In [54]:
df = pd.read_csv(PATH/'tmp.csv', index_col=0, header=0, dtype=str)
In [56]:
fnames = df.index.values
# fnames = df.iloc[:,0].values
df.iloc[:,0] = df.iloc[:,0].str.split(' ')
fnames,csv_labels= sorted(fnames), list(df.to_dict().values())[0]
In [60]:
fnames[:10], list(csv_labels.items())[:10]
Out[60]:
In [44]:
learner = get_learner(wrn_22(), 512)
In [45]:
x,y = next(iter(learner.data.trn_dl))
In [46]:
type(x[0]), type(y[0])
Out[46]:
In [21]:
next(iter(learner.data.val_dl))[0].type(), next(iter(learner.data.trn_dl))[0].type()
Out[21]:
In [23]:
tfms = tfms_from_stats(stats, sz=32)
md = ImageClassifierData.from_csv(PATH, 'train', PATH/'tmp.csv', tfms=tfms)
In [26]:
next(iter(md.trn_dl))[0].type()
Out[26]:
In [ ]:
In [47]:
learner = get_learner(wrn_22(), 512)
learner.lr_find(wds=1e-4)
learner.sched.plot(n_skip_end=1)
TypeError: eq received an invalid combination of arguments - got (torch.LongTensor), but expected one of:
* (int value)
didn't match because some of the arguments have invalid types: (torch.LongTensor)
* (torch.cuda.LongTensor other)
didn't match because some of the arguments have invalid types: (torch.LongTensor)
In [ ]:
learner = get_learner(wrn_22(), 512)
learner.lr_find(wds=1e-4)
learner.sched.plot(n_skip_end=1)