In [ ]:
# 0-coastal 1
# 1-blue 2
# 2-green 3
# 3-yellow 4
# 4-red 5
# 5-red edge 6
# 6-near-IR1 7
# 7-near-IR2 8
In [ ]:
from __future__ import division
import pandas as pd
import cv2
from helpers import Helper
import numpy as np
import random
import tifffile as tiff
import matplotlib.pyplot as plt
import shapely
import os
%matplotlib inline
import matplotlib.pyplot as plt
from skimage.transform import rescale, resize
def calc_jac(pred, label):
itersec = np.multiply(pred, label).sum()
summ = (pred+label).sum()
jac = itersec/(summ-itersec)
return jac
def dilation(binary, kernel_type):
return cv2.dilate(binary , kernel_type, iterations = 5)
def erosion(binary, kernel_type):
return cv2.erode(binary, kernel_type, iterations = 5)
def ellipse_open(binary, kernel_type):
return cv2.morphologyEx(binary, cv2.MORPH_OPEN, kernel_type)
def best_morphology(binary, label):
# delete noise
# noise removal
kernel = np.ones((5,5),np.uint8)
opening = cv2.morphologyEx(binary, cv2.MORPH_CLOSE, kernel, iterations = 2)
# kernel_size = 5
# kernel = np.ones((kernel_size, kernel_size),np.uint8)
# opening = cv2.morphologyEx(binary, cv2.MORPH_OPEN, kernel)
# test best morphology
kernel_size = 3
kernel_type = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(kernel_size, kernel_size))
jac_d = calc_jac(opening, label)
jac_e = calc_jac(erosion(opening, kernel_type), label)
jac_ellipse_open = calc_jac(ellipse_open(opening, kernel_type), label)
morph = [dilation, erosion, ellipse_open]
ind = np.argmax([jac_d, jac_e, jac_ellipse_open])
return morph[ind], kernel_type
def normalize(bands):
out = np.zeros_like(bands).astype(np.float32)
n = bands.shape[2]
print bands.shape
for i in range(n):
out[:, :, i] = equalize_adapthist((bands[:,:,i]).astype(np.float32))
return out.astype(np.float32)
def stretch_n(bands, lower_percent=0, higher_percent=100):
out = np.zeros_like(bands).astype(np.float32)
n = bands.shape[2]
for i in range(n):
a = 0 # np.min(band)
b = 1 # np.max(band)
c = np.percentile(bands[:, :, i], lower_percent)
d = np.percentile(bands[:, :, i], higher_percent)
t = a + (bands[:, :, i] - c) * (b - a) / (d - c)
t[t < a] = a
t[t > b] = b
out[:, :, i] = t
return out.astype(np.float32)
In [ ]:
data = pd.read_csv('../input/train_wkt_v4.csv')
data = data[data.MultipolygonWKT != 'MULTIPOLYGON EMPTY']
grid_sizes_fname = '../input/grid_sizes.csv'
wkt_fname = '../input/train_wkt_v4.csv'
image_fname = '../input/three_band/'
In [ ]:
helper = Helper()
pr_size = 512
for IM_ID in data[data.ClassType == 10].ImageId.values:
print IM_ID
rgb, mask = helper.load_im_polymask(IM_ID, '10', image_fname, wkt_fname, grid_sizes_fname)
m = tiff.imread('../input/M/train/{}.tif'.format(IM_ID))
NIR=resize(m[6,:,:], (rgb.shape[0], rgb.shape[1]))[:,:, np.newaxis]
NIR2=resize(m[7,:,:], (rgb.shape[0], rgb.shape[1]))[:,:, np.newaxis]
B=resize(m[1,:,:], (rgb.shape[0], rgb.shape[1]))[:,:, np.newaxis]
G=resize(m[2,:,:], (rgb.shape[0], rgb.shape[1]))[:,:, np.newaxis]
R=resize(m[4,:,:], (rgb.shape[0], rgb.shape[1]))[:,:, np.newaxis]
C=resize(m[0,:,:], (rgb.shape[0], rgb.shape[1]))[:,:, np.newaxis]
Y=resize(m[3,:,:], (rgb.shape[0], rgb.shape[1]))[:,:, np.newaxis]
RE=resize(m[5,:,:], (rgb.shape[0], rgb.shape[1]))[:,:, np.newaxis]
bands = np.concatenate([C, B, G, Y,R, RE, NIR, NIR2], axis=2)
rgb = bands
counter = 0
for i in range(0,6):
for j in range(0,6):
crop = rgb[i*pr_size:i*pr_size+pr_size, j*pr_size:j*pr_size+pr_size, :]
msk = mask[i*pr_size:i*pr_size+pr_size, j*pr_size:j*pr_size+pr_size]
tiff.imsave('data/train/A/{}.tif'.format(IM_ID+str(i)+str(j)), crop)
tiff.imsave('data/train/B/{}.tif'.format(IM_ID+str(i)+str(j)), msk)
counter+=1
crop = rgb[i*pr_size:i*pr_size+pr_size, -pr_size:, :]
msk = mask[i*pr_size:i*pr_size+pr_size, -pr_size:]
tiff.imsave('data/train/A/{}.tif'.format(IM_ID+str(i)+str(j)+'right'), crop)
tiff.imsave('data/train/B/{}.tif'.format(IM_ID+str(i)+str(j)+'right'), msk)
counter+=1
# predictions for bottom part
if i==5:
for j in range(0,6):
crop = rgb[-pr_size:, j*pr_size:j*pr_size+pr_size, :]
msk= mask[-pr_size:, j*pr_size:j*pr_size+pr_size]
tiff.imsave('data/train/A/{}.tif'.format(IM_ID+str(i+1)+str(j)), crop)
tiff.imsave('data/train/B/{}.tif'.format(IM_ID+str(i+1)+str(j)), msk)
counter+=1
crop = rgb[-pr_size:, -pr_size:, :]
msk = mask[-pr_size:, -pr_size:]
tiff.imsave('data/train/A/{}.tif'.format(IM_ID+str(i+1)+str(j+1)), crop)
tiff.imsave('data/train/B/{}.tif'.format(IM_ID+str(i+1)+str(j+1)), msk)
counter+=1
print counter
In [ ]:
helper = Helper()
for IM_ID in data[data.ClassType == 9].ImageId.values:
rgb, mask = helper.load_im_polymask(IM_ID, '9', image_fname, wkt_fname, grid_sizes_fname)
plt.imshow(stretch_n(rgb, lower_percent=2, higher_percent=98))
plt.show()
plt.imshow(mask)
plt.show()
In [ ]:
# mean and std
mean = np.empty((637, 512, 512, 8))
for i, IM_ID in enumerate(os.listdir('data/train/A/')):
img = tiff.imread('data/train/A/{}.tif'.format(IM_ID[:-4]))
mean[i, :,:, :] = img
print mean.shape
mean_img = np.mean(mean, axis=0)
tiff.imsave('data/mean_img.tif', mean_img)
In [ ]:
std_img = np.std(mean, axis=0)
tiff.imsave('data/std_img.tif', std_img)
In [ ]:
helper = Helper()
pr_size = 512
for IM_ID in data[data.ClassType == 10].ImageId.values[-1:]:
print IM_ID
rgb, mask = helper.load_im_polymask(IM_ID, '10', image_fname, wkt_fname, grid_sizes_fname)
m = tiff.imread('../input/M/train/{}.tif'.format(IM_ID))
NIR = resize(m[6,:,:], (rgb.shape[0], rgb.shape[1]))[:,:, np.newaxis]
NIR2 = resize(m[7,:,:], (rgb.shape[0], rgb.shape[1]))[:,:, np.newaxis]
B = resize(m[1,:,:], (rgb.shape[0], rgb.shape[1]))[:,:, np.newaxis]
G = resize(m[2,:,:], (rgb.shape[0], rgb.shape[1]))[:,:, np.newaxis]
R = resize(m[4,:,:], (rgb.shape[0], rgb.shape[1]))[:,:, np.newaxis]
C = resize(m[0,:,:], (rgb.shape[0], rgb.shape[1]))[:,:, np.newaxis]
Y = resize(m[3,:,:], (rgb.shape[0], rgb.shape[1]))[:,:, np.newaxis]
RE = resize(m[5,:,:], (rgb.shape[0], rgb.shape[1]))[:,:, np.newaxis]
bands = np.concatenate([C, B, G, Y,R, RE, NIR, NIR2], axis=2)
rgb=bands
counter = 0
for i in range(0,6):
for j in range(0,6):
crop = rgb[i*pr_size:i*pr_size+pr_size, j*pr_size:j*pr_size+pr_size, :]
msk = mask[i*pr_size:i*pr_size+pr_size, j*pr_size:j*pr_size+pr_size]
tiff.imsave('data/val/A/{}.tif'.format(IM_ID+str(i)+str(j)), crop)
tiff.imsave('data/val/B/{}.tif'.format(IM_ID+str(i)+str(j)), msk)
counter+=1
crop = rgb[i*pr_size:i*pr_size+pr_size, -pr_size:, :]
msk = mask[i*pr_size:i*pr_size+pr_size, -pr_size:]
tiff.imsave('data/val/A/{}.tif'.format(IM_ID+str(i)+str(j)+'right'), crop)
tiff.imsave('data/val/B/{}.tif'.format(IM_ID+str(i)+str(j)+'right'), msk)
counter+=1
# predictions for bottom part
if i==5:
for j in range(0,6):
crop = rgb[-pr_size:, j*pr_size:j*pr_size+pr_size, :]
msk= mask[-pr_size:, j*pr_size:j*pr_size+pr_size]
tiff.imsave('data/val/A/{}.tif'.format(IM_ID+str(i+1)+str(j)), crop)
tiff.imsave('data/val/B/{}.tif'.format(IM_ID+str(i+1)+str(j)), msk)
counter+=1
crop = rgb[-pr_size:, -pr_size:, :]
msk = mask[-pr_size:, -pr_size:]
tiff.imsave('data/val/A/{}.tif'.format(IM_ID+str(i+1)+str(j+1)), crop)
tiff.imsave('data/val/B/{}.tif'.format(IM_ID+str(i+1)+str(j+1)), msk)
counter+=1
print counter
In [ ]:
import pandas as pd
In [ ]:
#sw = pd.read_csv('../submits/0.3137_standing_water_index.csv')
#road = pd.read_csv('../submits/road_p2p_final_no_water.csv')
# build = pd.read_csv('../submits/build_p2p_final.csv')
crops = pd.read_csv('../submits/0.6929_crops_pix2pix.csv')
man_made = pd.read_csv('../submits/manmade_final.csv')
waterway = pd.read_csv('../submits/0.9528_waterway_index.csv')
trees = pd.read_csv('../submits/0.4722_trees_pix2pix.csv')
tracks = pd.read_csv('../submits/tracks_p2p_final.csv')
vl = pd.read_csv('../submits/vl_final.csv')
In [ ]:
# main submit
df = pd.read_csv('../submits/1_with_water.csv')
In [ ]:
counter = 1
for idx, row in df.iterrows():
_id = row[0]
kls = row[1]
# if kls == 1:
# df.iloc[idx, 2] = build[(build.ClassType==kls) & (build.ImageId==_id)].MultipolygonWKT.values[0]
if kls == 2:
df.iloc[idx, 2] = man_made[(man_made.ClassType==kls) & (man_made.ImageId==_id)].MultipolygonWKT.values[0]
# if kls == 3:
# df.iloc[idx, 2] = road[(road.ClassType==kls) & (road.ImageId==_id)].MultipolygonWKT.values[0]
if kls == 4:
df.iloc[idx, 2] = tracks[(tracks.ClassType==kls) & (tracks.ImageId==_id)].MultipolygonWKT.values[0]
if kls == 5:
df.iloc[idx, 2] = trees[(trees.ClassType==kls) & (trees.ImageId==_id)].MultipolygonWKT.values[0]
# if kls == 6:
# df.iloc[idx, 2] = crops[(crops.ClassType==kls) & (crops.ImageId==_id)].MultipolygonWKT.values[0]
if kls == 7:
df.iloc[idx, 2] = waterway[(waterway.ClassType==kls) & (waterway.ImageId==_id)].MultipolygonWKT.values[0]
# if kls == 8:
# df.iloc[idx, 2] = sw[(sw.ClassType==kls) & (sw.ImageId==_id)].MultipolygonWKT.values[0]
if kls == 9:
df.iloc[idx, 2] = vl[(vl.ClassType==kls) & (vl.ImageId==_id)].MultipolygonWKT.values[0]
print counter
counter += 1
print df.head()
df.to_csv('final_submit_adversarial.csv', index=False)
In [ ]:
final = pd.read_csv('final_submit_adversarial.csv')
In [ ]:
6100_0_2 6060_1_4
In [ ]:
final[(final.ImageId == '6060_1_4') & (final.ClassType == 2)]
In [ ]:
final.iloc[1911, 2] = 'MULTIPOLYGON EMPTY'
In [ ]:
final.to_csv('final_submit_adversarial.csv', index=False)
In [ ]:
In [ ]: