In [41]:
import numpy as np
import cv2
from matplotlib import pyplot as plt
from utils import normalize, MyWarp, annotate, cosine
In [42]:
####### Question 1 - Affine Transformation
def affineRectify(name, custom_pts='none', plot=True):
file_loc = './data/q1/'+name+'.jpg'
# Save image in set directory
# Read RGB image
img = cv2.imread(file_loc)
img_line = cv2.imread(file_loc)
if type(custom_pts) == str:
points = q1_annotation.item().get(name)
else:
points = np.copy(custom_pts)
colors = [(255,0,0),(0,255,0)]
lines = []
for j in range(8):
if j < 4:
start_point = (int(points[2*j,0]),int(points[2*j,1]))
end_point = (int(points[2*j+1,0]),int(points[2*j+1,1]))
img_line = cv2.line(img_line, start_point, end_point, colors[j//2], 15)
x1 = np.array([points[2*j,0], points[2*j,1], 1])
x2 = np.array([points[2*j+1,0], points[2*j+1,1], 1])
lines.append(normalize(np.cross(x1, x2)))
x1 = np.cross(lines[0], lines[1])
x2 = np.cross(lines[2], lines[3])
l_inf = normalize(np.cross(normalize(x1), normalize(x2)))
H = np.array([[1,0,0], [0,1,0], [0,0,0]], dtype=float)
H[-1,:] = l_inf
H_invT = np.linalg.inv(H.T)
img_rect = MyWarp(img, H)
if plot:
plt.figure(figsize=(10, 10), dpi=80)
plt.subplot(1,3,1)
plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
plt.title("Input Image")
plt.axis('off')
plt.subplot(1,3,2)
plt.imshow(cv2.cvtColor(img_line, cv2.COLOR_BGR2RGB))
plt.title("Annotated Input Image")
plt.axis('off')
plt.subplot(1,3,3)
plt.imshow(cv2.cvtColor(img_rect, cv2.COLOR_BGR2RGB))
plt.title("Affine-Rectified Image")
plt.axis('off')
plt.show()
print('1.3 - ')
print(name+' Angles')
cos_before = [cosine(lines[4], lines[5]), cosine(lines[6], lines[7])]
print('Before: ',cos_before)
l_prime = [H_invT@line for line in lines]
cos_after = [cosine(l_prime[4], l_prime[5]), cosine(l_prime[6], l_prime[7])]
print('After: ',cos_after)
return H
In [43]:
names = ['book1', 'checker1', 'chess1', 'big_ben', 'alcala']
# names = ['book1', 'checker1', 'chess1', 'facade', 'tiles3', 'tiles5']
with open('./data/annotation/q1_annotation.npy','rb') as f:
q1_annotation = np.load(f, allow_pickle=True)
parallel_annotations = [[[667.5194805194801, 4976.502164502164, 1.0], [741.924242424242, 3447.822510822511, 1.0], [1628.0173160173154, 3150.2034632034633, 1.0], [1621.2532467532465, 4929.15367965368, 1.0], [802.8008658008653, 3021.686147186147, 1.0], [1445.3874458874452, 2798.4718614718613, 1.0], [836.6212121212116, 2379.099567099567, 1.0], [1404.80303030303, 2142.3571428571427, 1.0], [816.329004329004, 2047.6601731601731, 1.0], [1587.4329004329002, 1804.1536796536793, 1.0], [1228.9372294372288, 1290.0844155844152, 1.0], [1573.9047619047615, 1175.0952380952376, 1.0], [1857.9956709956705, 2182.9415584415583, 1.0], [1857.9956709956705, 2839.056277056277, 1.0], [2304.424242424242, 3204.316017316017, 1.0], [2263.839826839826, 2541.4372294372292, 1.0]],
[[123.99596774193552, 675.6451612903226, 1.0], [198.6935483870968, 387.3387096774193, 1.0], [1097.6854838709678, 383.4072580645161, 1.0], [1177.625, 670.4032258064516, 1.0], [448.9959677419356, 687.4395161290322, 1.0], [848.6935483870969, 684.8185483870968, 1.0], [515.8306451612904, 253.66935483870964, 1.0], [781.8588709677421, 251.04838709677415, 1.0], [804.1370967741937, 420.1008064516129, 1.0], [1092.4435483870968, 410.92741935483866, 1.0], [1092.4435483870968, 366.3709677419355, 1.0], [792.342741935484, 375.54435483870964, 1.0], [447.6854838709678, 675.6451612903226, 1.0], [462.1008064516129, 493.48790322580646, 1.0], [851.3145161290323, 676.9556451612902, 1.0], [829.0362903225808, 485.625, 1.0]]]
for i in range(len(names)):
name = names[i]
if i > 2:
file_loc = './data/q1/'+names[i]+'.jpg'
custom_pts = np.array([[pt[0], pt[1]] for pt in parallel_annotations[i-3]])
else:
custom_pts = 'none'
# print(custom_pts)
affineRectify(name, custom_pts, plot=True)
1.3 - book1 Angles Before: [0.9999895814383513, 0.9599844125720752] After: [0.9999934408698983, 0.9952989626474261]
1.3 - checker1 Angles Before: [0.8964448119431402, 0.9574522354365925] After: [0.9999943713107801, 0.9999729375269912]
1.3 - chess1 Angles Before: [0.9878470787639924, 0.9845941270659717] After: [0.9997703794618689, 0.9999978464776055]
1.3 - big_ben Angles Before: [0.9998740474835989, -0.9981310335652203] After: [0.9991839346467336, -0.9994515969800989]
1.3 - alcala Angles Before: [-0.9999992196491955, 0.9810692715118134] After: [-0.9999997245547617, 0.9999233989882037]
In [44]:
####### Question 2 - Metric Transformation
def metricRecitify(name, custom_pts='none', custom_parallel='none', plot=True):
H_aff = affineRectify(name, custom_parallel, plot=False)
file_loc = './data/q1/'+name+'.jpg'
img_new = cv2.imread(file_loc)
img_line = cv2.imread(file_loc)
if type(custom_pts) == str:
points = q2_annotation.item().get(name)
else:
points = np.copy(custom_pts)
colors = [(255,0,0),(0,255,0)]
lines = []
lines_aff = []
for j in range(8):
if j < 4:
start_point = (int(points[2*j,0]),int(points[2*j,1]))
end_point = (int(points[2*j+1,0]),int(points[2*j+1,1]))
img_line = cv2.line(img_line, start_point, end_point, colors[j//2], 15)
x1 = np.array([points[2*j,0], points[2*j,1], 1])
x2 = np.array([points[2*j+1,0], points[2*j+1,1], 1])
line = np.cross(normalize(x1), normalize(x2))
lines.append(line/line[2])
x1_aff = H_aff@np.array([points[2*j,0], points[2*j,1], 1])
x2_aff = H_aff@np.array([points[2*j+1,0], points[2*j+1,1], 1])
line = np.cross(normalize(x1_aff), normalize(x2_aff))
lines_aff.append(line/line[2])
img_line_rect = MyWarp(img_line, H_aff)
# lines_aff = [np.linalg.inv(H_aff.T)@line for line in lines]
C = np.array([[-lines_aff[0][1]*lines_aff[1][1]], [-lines_aff[2][1]*lines_aff[3][1]]])
A = np.array([[lines_aff[0][0]*lines_aff[1][0], lines_aff[0][0]*lines_aff[1][1]+lines_aff[0][1]*lines_aff[1][0]],
[lines_aff[2][0]*lines_aff[3][0], lines_aff[2][0]*lines_aff[3][1]+lines_aff[2][1]*lines_aff[3][0]]])
S = np.linalg.inv(A)@C
S = np.array([[S[0,0],S[1,0]],[S[1,0],1]])
U, D, Vt = np.linalg.svd(S)
A = U@np.diag(np.sqrt(D))@Vt
H = np.array([[A[0][0], A[0][1], 0], [A[1][0], A[1][1], 0], [0, 0, 1]])
H_metric = np.linalg.inv(H)@H_aff
# print(H_metric)
H_invT = np.linalg.inv(H_metric.T)
img_metric = MyWarp(img_new, H_metric)
if plot:
plt.figure(figsize=(10, 10), dpi=80)
plt.subplot(1,4,1)
plt.imshow(cv2.cvtColor(img_new, cv2.COLOR_BGR2RGB))
plt.title("Input Image")
plt.axis('off')
plt.subplot(1,4,2)
plt.imshow(cv2.cvtColor(img_line, cv2.COLOR_BGR2RGB))
plt.title("Annotated Input Image")
plt.axis('off')
plt.subplot(1,4,3)
plt.imshow(cv2.cvtColor(img_line_rect, cv2.COLOR_BGR2RGB))
plt.title("Affine-Rectified Image")
plt.axis('off')
plt.subplot(1,4,4)
plt.imshow(cv2.cvtColor(img_metric, cv2.COLOR_BGR2RGB))
plt.title("Metric-Rectified Image")
plt.axis('off')
plt.show()
print('2.3 - ')
print(name+' Angles')
cos_before = [cosine(lines[4], lines[5]), cosine(lines[6], lines[7])]
print('Before: ',cos_before)
l_prime = [H_invT@line for line in lines]
cos_after = [cosine(l_prime[4], l_prime[5]), cosine(l_prime[6], l_prime[7])]
print('After: ',cos_after)
return None
In [45]:
with open('./data/annotation/q2_annotation.npy','rb') as f:
q2_annotation = np.load(f, allow_pickle=True)
perp_annotations = [[[809.5649350649346, 2981.1017316017314, 1.0], [1452.151515151515, 2737.595238095238, 1.0], [1465.6796536796528, 2108.5367965367964, 1.0], [1458.915584415584, 2778.1796536796537, 1.0], [809.5649350649346, 3008.1580086580084, 1.0], [1465.6796536796528, 2108.5367965367964, 1.0], [850.1493506493503, 2372.335497835498, 1.0], [1452.151515151515, 2730.8311688311687, 1.0], [1871.5238095238092, 2805.2359307359307, 1.0], [2277.3679653679646, 2534.67316017316, 1.0], [2290.8961038961033, 3123.147186147186, 1.0], [1885.0519480519479, 2182.9415584415583, 1.0], [694.575757575757, 4333.915584415585, 1.0], [1600.9610389610389, 4056.5887445887447, 1.0], [687.8116883116882, 4333.915584415585, 1.0], [741.924242424242, 3359.8896103896104, 1.0]],
[[587.9072580645162, 683.508064516129, 1.0], [586.5967741935484, 490.866935483871, 1.0], [590.5282258064517, 679.5766129032259, 1.0], [703.2298387096776, 676.9556451612902, 1.0], [293.04838709677426, 497.41935483870964, 1.0], [236.6975806451613, 447.6209677419355, 1.0], [227.52419354838716, 500.0403225806451, 1.0], [302.2217741935484, 450.241935483871, 1.0], [207.866935483871, 367.6814516129032, 1.0], [125.30645161290326, 682.1975806451612, 1.0], [206.55645161290326, 383.4072580645161, 1.0], [1085.891129032258, 366.3709677419355, 1.0], [646.8790322580646, 206.49193548387098, 1.0], [646.8790322580646, 676.9556451612902, 1.0], [141.03225806451618, 680.8870967741935, 1.0], [1160.5887096774193, 671.7137096774193, 1.0]]]
for i in range(len(names)):
name = names[i]
if i > 2:
file_loc = './data/q1/'+names[i]+'.jpg'
custom_pts = np.array([[pt[0], pt[1]] for pt in perp_annotations[i-3]])
custom_parallel = np.array([[pt[0], pt[1]] for pt in parallel_annotations[i-3]])
else:
custom_pts, custom_parallel = 'none', 'none'
# print(custom_pts)
metricRecitify(name, custom_pts, custom_parallel, plot=True)
2.3 - book1 Angles Before: [0.11870842546128174, -0.16023064331361753] After: [-0.028819111972334697, 0.016234908527699973]
2.3 - checker1 Angles Before: [0.2526709009836574, 0.08817394212374821] After: [-0.0018168249234062417, 0.008198536236441218]
2.3 - chess1 Angles Before: [-0.6685881410241947, 0.04479521389935166] After: [0.021182064318718906, 0.00964023413980748]
2.3 - big_ben Angles Before: [0.17953024310030655, 0.3451733316175541] After: [0.00662871647077289, 0.07283810323423788]
2.3 - alcala Angles Before: [0.2725861740150785, 0.008997065140273144] After: [0.024049450753685454, 0.014218875331173875]
In [46]:
####### Question 3 - Planar Homography from Point Correspondences
def correspHomography(points, file_1, file_2, plot=False):
if type(file_2) == str:
img_2 = cv2.imread(file_2)
img_2_annotate = cv2.imread(file_2)
else:
img_2 = np.copy(file_2)
img_2_annotate = np.copy(file_2)
img_1 = cv2.imread(file_1)
points_2 = np.array([[0,0,1], [img_1.shape[1]-1,0,1], [img_1.shape[1]-1,img_1.shape[0]-1,1],[0,img_1.shape[0]-1,1]])
A = np.zeros((8,9))
for i in range(4):
A[i*2,3:6] = -points[i, 2]*points_2[i]
A[i*2,6:] = points[i, 1]*points_2[i]
A[i*2+1,:3] = points[i, 2]*points_2[i]
A[i*2+1,6:] = -points[i, 0]*points_2[i]
img_2_annotate = cv2.circle(img_2_annotate, (int(points[i,0]),int(points[i,1])), radius=10, color=(0,0,255), thickness=-1)
cv2.putText(img_2_annotate, str(i+1), (int(points[i,0])+25,int(points[i,1])+25), fontFace = cv2.FONT_HERSHEY_PLAIN,fontScale = 3,color = (0,0,255))
_,_,Vt = np.linalg.svd(A)
h = Vt[-1]/np.linalg.norm(Vt[-1])
H = h.reshape((3,3))
new_img_1 = cv2.warpPerspective(img_1, H, (img_2.shape[1], img_2.shape[0]))
gray_warp = cv2.cvtColor(new_img_1, cv2.COLOR_BGR2GRAY)
_, mask = cv2.threshold(gray_warp, 1, 255, cv2.THRESH_BINARY)
# Invert the mask for the background
mask_inv = cv2.bitwise_not(mask)
background = cv2.bitwise_and(img_2, img_2, mask=mask_inv)
# Extract the region of the warped image using the mask
foreground = cv2.bitwise_and(new_img_1, new_img_1, mask=mask)
# Combine the two images
img_full = cv2.add(background, foreground)
if plot:
plt.figure(figsize=(40, 40), dpi=180)
plt.subplot(1,4,1)
plt.imshow(cv2.cvtColor(img_1, cv2.COLOR_BGR2RGB))
plt.title("Foreground Image")
plt.axis('off')
plt.subplot(1,4,2)
plt.imshow(cv2.cvtColor(img_2, cv2.COLOR_BGR2RGB))
plt.title("Background Image")
plt.axis('off')
plt.subplot(1,4,3)
plt.imshow(cv2.cvtColor(img_2_annotate, cv2.COLOR_BGR2RGB))
plt.title("Annotated Background")
plt.axis('off')
plt.subplot(1,4,4)
plt.imshow(cv2.cvtColor(img_full, cv2.COLOR_BGR2RGB))
plt.title("Combined Image")
plt.axis('off')
plt.show()
return img_full
In [47]:
with open('./data/annotation/q3_annotation.npy','rb') as f:
q3_annotation = np.load(f, allow_pickle=True)
points = list(q3_annotation.item().get('desk-perspective'))
points = np.array([[point[0], point[1],1] for point in points])
file_1 = './data/q3/'+'desk-normal'+'.png'
file_2 = './data/q3/'+'desk-perspective'+'.png'
img_1 = correspHomography(points, file_1, file_2, plot=True)
img_new = cv2.imread('./data/q5/'+'multi_screen_laptop'+'.jpg')
background_corresp = [[53.83522727272731, 371.42207792207773, 1.0], [474.76055194805207, 407.5462662337661, 1.0], [463.7662337662338, 641.5681818181818, 1.0], [52.264610389610425, 603.8733766233765, 1.0]]
points = np.array(background_corresp)
img_2 = correspHomography(points, './data/q5/'+'cat'+str(1)+'.jpg',img_new, plot=True)
In [48]:
def metricRectify_five(points, name, plot=True):
def A_row(l1, l2):
row = np.zeros((1,6))
row[0,0] = l1[0]*l2[0]
row[0,1] = (l1[0]*l2[1]+l1[1]*l2[0])/2
row[0,2] = l1[1]*l2[1]
row[0,3] = (l1[0]*l2[2]+l1[2]*l2[0])/2
row[0,4] = (l1[1]*l2[2]+l1[2]*l2[1])/2
row[0,5] = l1[2]*l2[2]
return row
def reconstruct_IAC(c):
return np.array([[c[0],c[1]/2,c[3]/2],[c[1]/2,c[2],c[4]/2],[c[3]/2,c[4]/2,c[5]]])
file_loc = './data/q1/'+name+'.jpg'
img_new = cv2.imread(file_loc)
img_line = cv2.imread(file_loc)
colors = [(255,0,0),(0,255,0), (0,0,255),(255,255,0),(255,0,255),(255,255,255),(0,255,255),(125,255,125)]
lines = []
for j in range(len(points)//2):
start_point = (int(points[2*j,0]),int(points[2*j,1]))
end_point = (int(points[2*j+1,0]),int(points[2*j+1,1]))
img_line = cv2.line(img_line, start_point, end_point, colors[j//2], 15)
x1 = normalize(points[2*j])
x2 = normalize(points[2*j+1])
line = np.cross(x1,x2)
lines.append(line/line[2])
A = np.zeros((len(points)//4,6))
for j in range(len(points)//4):
row = A_row(lines[2*j],lines[2*j+1])
A[j,:] = row
_,_,Vt = np.linalg.svd(A)
c = Vt[-1,:]
C = reconstruct_IAC(c)
U,S,Vt = np.linalg.svd(C)
S = np.diag([S[0],S[1],1])
# S_ = np.diag([np.sqrt(S[0]),np.sqrt(S[1]),10])@np.diag([1,1,S[2]/100])@np.diag([np.sqrt(S[0]),np.sqrt(S[1]),10])
H_metric = (np.linalg.inv(np.sqrt(S)))@Vt
H_invT = np.linalg.inv(H_metric.T)
img_metric = MyWarp(img_new, H_metric)
if plot:
plt.figure(figsize=(10, 10), dpi=80)
plt.subplot(1,3,1)
plt.imshow(cv2.cvtColor(img_new, cv2.COLOR_BGR2RGB))
plt.title("Input Image")
plt.axis('off')
plt.subplot(1,3,2)
plt.imshow(cv2.cvtColor(img_line, cv2.COLOR_BGR2RGB))
plt.title("Annotated Input Image")
plt.axis('off')
plt.subplot(1,3,3)
plt.imshow(cv2.cvtColor(img_metric, cv2.COLOR_BGR2RGB))
plt.title("Metric-Rectified Image")
plt.axis('off')
plt.show()
print('4.3 - ')
print(name+' Angles')
cos_before = [cosine(lines[4], lines[5]), cosine(lines[6], lines[7]), cosine(lines[8], lines[9])]
print('Before: ',cos_before)
l_prime = [H_invT@line for line in lines]
cos_after = [cosine(l_prime[4], l_prime[5]), cosine(l_prime[6], l_prime[7]), cosine(l_prime[2], l_prime[3])]
print('After: ',cos_after)
return None
In [49]:
names = ['checker1','alcala']
for i in range(len(names)):
name = names[i]
file_loc = './data/q1/'+names[i]+'.jpg'
perp_lines = [[[76.18560606060609, 257.5, 1.0], [115.27651515151518, 165.47348484848476, 1.0], [201.60227272727278, 206.1931818181818, 1.0], [-0.3674242424242209, 209.45075757575756, 1.0],[685.3522727272727, 307.17803030303025, 1.0], [532.2462121212121, 249.35606060606057, 1.0], [682.094696969697, 306.3636363636363, 1.0], [735.844696969697, 246.09848484848482, 1.0], [664.9924242424242, 105.20833333333326, 1.0], [732.5871212121212, 163.030303030303, 1.0], [762.719696969697, 132.89772727272725, 1.0], [625.0871212121212, 129.6401515151515, 1.0], [618.5719696969697, 576.7424242424242, 1.0], [783.0795454545455, 564.5265151515151, 1.0], [667.4356060606061, 505.07575757575756, 1.0], [705.7121212121212, 568.5984848484849, 1.0]],
[[1110.7903225806451, 676.9556451612902, 1.0], [1076.717741935484, 534.1129032258065, 1.0], [1076.717741935484, 534.1129032258065, 1.0], [1012.5040322580645, 534.1129032258065, 1.0], [125.30645161290326, 680.8870967741935, 1.0], [201.31451612903228, 378.1653225806451, 1.0], [201.31451612903228, 378.1653225806451, 1.0], [1088.5120967741937, 371.61290322580646, 1.0], [230.14516129032265, 500.0403225806451, 1.0], [302.2217741935484, 448.9314516129032, 1.0], [243.25000000000003, 448.9314516129032, 1.0], [291.73790322580646, 500.0403225806451, 1.0], [1003.3306451612904, 492.17741935483866, 1.0], [1059.6814516129032, 438.4475806451612, 1.0], [994.1572580645162, 442.3790322580645, 1.0], [1068.8548387096773, 486.9354838709677, 1.0], [531.5564516129033, 688.75, 1.0], [545.9717741935484, 354.57661290322574, 1.0], [545.9717741935484, 354.57661290322574, 1.0], [759.5806451612904, 341.4717741935484, 1.0], [563.0080645161291, 868.2862903225806, 1.0], [637.7056451612904, 688.75, 1.0], [635.0846774193549, 688.75, 1.0], [123.99596774193552, 686.1290322580645, 1.0], [175.10483870967747, 503.9717741935484, 1.0], [137.10080645161295, 629.7782258064516, 1.0], [137.10080645161295, 629.7782258064516, 1.0], [181.65725806451618, 632.3991935483871, 1.0]]]
custom_pts = perp_lines[i]
# new_pts = annotate(file_loc)
# print(new_pts)
points = []
if i == 0:
points = list(q2_annotation.item().get(name))
[points.append(cstm_pt) for cstm_pt in custom_pts]
# [points.append(cstm_pt) for cstm_pt in new_pts]
points = np.array([[point[0],point[1],1] for point in points])
# print(custom_pts)
metricRectify_five(points, name, plot=True)
4.3 - checker1 Angles Before: [0.2526709009836574, 0.08817394212374821, 0.40576061177161965] After: [0.05156252622779721, 0.05251846911485593, 0.06304216371795528]
4.3 - alcala Angles Before: [0.14180759183203495, -0.2680493005056784, 0.10419397008629556] After: [-0.03696279404669118, -0.07272786418332218, -0.047736663009790514]
In [50]:
####### Question 5 - Multiple Correspondence
img_back = cv2.imread('./data/q5/'+'multi_screen_laptop'+'.jpg')
img_new = cv2.imread('./data/q5/'+'multi_screen_laptop'+'.jpg')
background_corresp = [[53.83522727272731, 371.42207792207773, 1.0], [474.76055194805207, 407.5462662337661, 1.0], [463.7662337662338, 641.5681818181818, 1.0], [52.264610389610425, 603.8733766233765, 1.0], [509.3141233766234, 383.9870129870128, 1.0], [963.2224025974026, 454.66477272727263, 1.0], [930.2394480519481, 789.2061688311687, 1.0], [485.7548701298702, 702.8222402597402, 1.0], [1008.7702922077922, 493.9301948051947, 1.0], [1366.8709415584417, 586.5965909090908, 1.0], [1333.887987012987, 866.166396103896, 1.0], [978.9285714285716, 740.5170454545454, 1.0]]
points = np.array(background_corresp)
for i in range(3):
img_new = correspHomography(points[4*i:(4*i+4)], './data/q5/'+'cat'+str(i+1)+'.jpg',img_new, plot=False)
plt.figure(figsize=(20, 20), dpi=80)
plt.subplot(1,3,1)
plt.imshow(cv2.cvtColor(img_back, cv2.COLOR_BGR2RGB))
plt.title("Input Image")
plt.axis('off')
plt.subplot(1,3,2)
plt.imshow(cv2.cvtColor(img_new, cv2.COLOR_BGR2RGB))
plt.title("Annotated Input Image")
plt.axis('off')
plt.show()
