In [31]:
from imagerie import *
from scipy.misc import toimage, imsave
from scipy.ndimage.filters import laplace
from scipy.ndimage import uniform_filter, gaussian_filter
%matplotlib inline
plt.rcParams['figure.figsize'] = (14, 8)
In [11]:
flyfront = imread("fly-front.jpg")
flyback = imread("fly-back.jpg")
flyref = imread("fly-reference.jpg")
In [12]:
fig = plt.figure()
fig.add_subplot(1,3,1)
plt.imshow(flyfront)
fig.add_subplot(1,3,2)
plt.imshow(flyback)
fig.add_subplot(1,3,3)
plt.imshow(flyref)
Out[12]:
In [15]:
# grey image only for initial testing
im = rgb2gray(flyfront)
In [20]:
average_filter_size = 31
base_layer = uniform_filter(im, size=average_filter_size)
detail_layer = im - base_layer
In [21]:
fig = plt.figure()
fig.add_subplot(1, 2, 1)
plt.imshow(base_layer, cmap='gray')
fig.add_subplot(1, 2, 2)
plt.imshow(detail_layer, cmap='gray')
Out[21]:
In [24]:
sigma_r = 5
saliency = gaussian_filter(abs(laplace(im)), sigma_r)
In [25]:
imshow(saliency)
Out[25]:
In [26]:
im2 = rgb2gray(flyback)
saliency2 = gaussian_filter(abs(laplace(im2)), sigma_r)
imshow(saliency2)
Out[26]:
In [30]:
mask = np.argmax([saliency, saliency2], axis=0)
In [32]:
imshow(mask * 255)
Out[32]:
In [35]:
imshow(im2 * mask + im * (1 - mask))
Out[35]:
In [4]:
def extract_windows(A, size, padding=True):
# inspired by http://stackoverflow.com/questions/30109068/implement-matlabs-im2col-sliding-in-python
if padding:
before = int((size - 1) / 2)
after = size - 1 - before
A = np.pad(A, ((before, after), (before, after)), 'mean')
M,N = A.shape
B = [M - size + 1, N - size + 1]
col_extent = N - B[1] + 1
row_extent = M - B[0] + 1
start_idx = np.arange(B[0])[:,None]*N + np.arange(B[1])
offset_idx = np.arange(row_extent)[:,None]*N + np.arange(col_extent)
return np.take (A,start_idx.ravel()[:,None] + offset_idx.ravel())
In [5]:
def guided_filtering(im, guide, size=3, epsilon=0.1):
if size % 2 != 1:
raise ArgumentError('Size must be an odd number.')
if len(guide.shape) >= 3:
return sum([guided_filtering(im, guide[:,:,k], size, epsilon) for k in range(guide.shape[2])])
# extract windows
im_wins = extract_windows(im, size)
guide_wins = extract_windows(guide, size)
mu_k = guide_wins.mean(axis=1)
nu_k = im_wins.mean(axis=1)
delta_k = guide_wins.std(axis=1)
a_k = (im_wins * guide_wins).mean(axis=1) - mu_k * nu_k / (delta_k + epsilon)
b_k = nu_k - a_k * mu_k
a_wins = extract_windows(a_k.reshape(*im.shape), size)
b_wins = extract_windows(b_k.reshape(*im.shape), size)
return a_wins.mean(axis=1).reshape(*im.shape) * guide + b_wins.mean(axis=1).reshape(*im.shape)
In [131]:
g = guided_filtering(1 - mask, im / 255, size=5, epsilon=0.1)
In [132]:
plt.imshow(g, cmap='gray', clim=(0,1))
Out[132]:
In [133]:
g2 = guided_filtering(mask, im2 / 255, size=5, epsilon=0.1)
In [134]:
plt.imshow(g2, cmap='gray', clim=(0,1))
Out[134]:
In [135]:
base_layer2 = uniform_filter(im2, size=average_filter_size)
detail_layer2 = im2 - base_layer2
In [141]:
fused_base = (base_layer * g + base_layer2 * g2) / (g + g2)
fused_detail = (detail_layer * g + detail_layer2 * g2) / (g + g2)
In [142]:
imshow(fused_base)
imshow(fused_detail)
imshow(fused_base + fused_detail)
Out[142]:
We now sum it up into a single procedure:
In [16]:
def extract_maps(im, average_filter_size=31, sigma_r=5):
base_layer = uniform_filter(im, size=average_filter_size)
detail_layer = im - base_layer
if len(im.shape) >= 3:
imgray = rgb2gray(im)
else:
imgray = im.astype(np.float64) - 0.0001
saliency = gaussian_filter(abs(laplace(imgray)), sigma_r)
return base_layer, detail_layer, saliency
In [17]:
def GFF(images,
average_filter_size = 31,
sigma_r=5,
r_base=5,
epsilon_base=2,
r_detail=5,
epsilon_detail=0.1):
base_layers, detail_layers, saliencies = zip(*[extract_maps(im, average_filter_size, sigma_r) for im in images])
saliency_idx = np.argmax(saliencies, axis=0)
masks = [saliency_idx == i for i in range(len(images))]
base_weights = [guided_filtering(mask, im / 255, size=r_base, epsilon=epsilon_base) for im, mask in zip(images, masks)]
detail_weights = [guided_filtering(mask, im / 255, size=r_detail, epsilon=epsilon_detail) for im, mask in zip(images, masks)]
# a bit hacky
if len(images[0].shape) >= 3:
base_weights = [w[:,:,None] for w in base_weights]
detail_weights = [w[:,:,None] for w in detail_weights]
base = sum([base * weight for base, weight in zip(base_layers, base_weights)]) / sum(base_weights)
detail = sum([detail * weight for detail, weight in zip(detail_layers, base_weights)]) / sum(base_weights)
return detail + base
In [185]:
fused = GFF([im, im2])
In [186]:
imshow(fused)
Out[186]:
In [295]:
fly = GFF([flyfront.astype(np.float64), flyback.astype(np.float64)]).astype(np.uint8)
In [296]:
imshow(fly)
Out[296]:
In [297]:
imshow(flyref)
Out[297]:
In [273]:
def imarrayshow(a):
n = len(a)
fig = plt.figure()
for k in range(n):
fig.add_subplot(1, n, k+1)
plt.imshow(a[k], cmap='gray')
plt.show()
In [274]:
imarrayshow(images)
imarrayshow(saliencies)
imarrayshow(masks)
In [275]:
imarrayshow(detail_weights)
imarrayshow(detail_layers)
Bookshelf¶
In [22]:
b = {}
for i in range(1, 8):
b[i] = imread("image/bookshelf%d-small.jpg" % i)
In [25]:
def cGFF(images, *args, **kwargs):
return GFF([i.astype(np.float64) for i in images], *args, **kwargs).astype(np.uint8)
In [34]:
b12 = cGFF([b[1], b[2]])
In [19]:
imshow(b12)
Out[19]:
In [21]:
imshow(b2)
Out[21]:
In [26]:
b_full = cGFF([b[1], b[2], b[3], b[4], b[5], b[6], b[7]])
In [27]:
imshow(b_full)
Out[27]:
In [36]:
toimage(b12, cmin=0, cmax=256).save('output/bookshelf1-2-small.jpg')
In [37]:
toimage(b_full, cmin=0, cmax=256).save('output/bookshelf-full-small.jpg')
In [ ]: