import cv2
import matplotlib.pyplot as plt
import numpy as np
import pytorch3d
import pytorch3d.io
from pytorch3d.vis.plotly_vis import plot_scene
from tqdm.auto import tqdm

from starter.utils import *
import os
import sys
import torch

# This should print True if you are using your GPU
print("Using GPU:", torch.cuda.is_available())
if torch.cuda.is_available():
    device = torch.device("cuda:0")
else:
    device = torch.device("cpu")

Using GPU: False

renderer = get_mesh_renderer(image_size=512)

vertices, faces = load_cow_mesh(path="data/cow.obj")
vertices=vertices.unsqueeze(0)
faces=faces.unsqueeze(0)

print("Vertices", vertices.shape,vertices.dtype)
print("Faces", faces.shape,faces.dtype)

Vertices torch.Size([1, 2930, 3]) torch.float32
Faces torch.Size([1, 5856, 3]) torch.int64

"""textures=vertices.clone()
textures=(textures - textures.min())/(textures.max()-textures.min())
textures=textures.norm(dim=2,keepdim=True)
textures = pytorch3d.renderer.TexturesVertex(textures)"""
texture_rgb = torch.ones_like(vertices) # N X 3
texture_rgb = texture_rgb * torch.tensor([0.7, 0.7, 1])
textures = pytorch3d.renderer.TexturesVertex(texture_rgb)

meshes = pytorch3d.structures.Meshes(
    verts=vertices,
    faces=faces,
    textures=textures,
)

cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R=torch.eye(3).unsqueeze(0),
    T=torch.tensor([[0, 0, 3]]),
    fov=60,
)

image_size = 512

raster_settings = pytorch3d.renderer.RasterizationSettings(image_size=image_size)
rasterizer = pytorch3d.renderer.MeshRasterizer(
    raster_settings=raster_settings,
)
shader = pytorch3d.renderer.HardPhongShader(device=device)
renderer = pytorch3d.renderer.MeshRenderer(
    rasterizer=rasterizer,
    shader=shader,
)

lights = pytorch3d.renderer.PointLights(location=[[0, 0, -3]])
rend = renderer(meshes, cameras=cameras, lights=lights)
image = rend[0, ..., :3].numpy()
print(image.shape)

(512, 512, 3)

plt.imshow(image)

<matplotlib.image.AxesImage at 0x162639550>

texture_rgb = vertices.clone()
texture_rgb = (texture_rgb - texture_rgb.min()) / (texture_rgb.max() - texture_rgb.min())
texture_rgb /= texture_rgb.norm(dim=2, keepdim=True)
textures_rainbow = pytorch3d.renderer.TexturesVertex(texture_rgb.to(device))
meshes.textures = textures_rainbow

image = renderer(meshes, cameras=cameras, lights=lights)
plt.imshow(image[0].cpu().numpy())

<matplotlib.image.AxesImage at 0x16229d7c0>

num_views = 36
R, T = pytorch3d.renderer.look_at_view_transform(
    dist=5,
    elev=0,
    azim=np.linspace(0, 360, num_views, endpoint=False),
)
many_cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R=R,
    T=T,
    device=device
)
images = renderer(meshes.extend(num_views), cameras=many_cameras, lights=lights)

images.shape

torch.Size([36, 512, 512, 4])

images=(images.cpu().numpy()*255).astype(np.uint8)

import imageio
duration=1000//15
imageio.mimsave('rbg_cow.gif',images,duration=duration,loop=0)

1.1. 360-degree Renders (5 points)

1.2 Re-creating the Dolly Zoom (10 points)

%run dolly_zoom.py

  0%|          | 0/25 [00:00<?, ?it/s]

Tetrahedron

Consider the side of the traingles to be of length root(2) and the traingles to be equilateral.

To form a tetrahedron, we start with four vertices arranged in 3D space so their centroid, is at the origin (0,0,0). The structure's four triangular faces are then created by connecting every possible set of three vertices.

vertices = torch.tensor(
    [[0.5, 0.5, 0.5],
     [0.5, -0.5, -0.5],
     [-0.5, 0.5, -0.5],
     [-0.5, -0.5, 0.5]]
).to(dtype=torch.float32)

faces = torch.tensor(
    [[0, 1, 2],
     [1, 2, 3],
     [2, 3, 0],
     [0, 1, 3]],
    dtype=torch.int64
)
print("vertices: ", vertices[0])
print("faces: ", faces[0])
vertices = vertices.unsqueeze(0)  # 1 x N_v x 3
faces = faces.unsqueeze(0)
print("Vertices", vertices.shape,vertices.dtype)
print("Faces", faces.shape,faces.dtype)

vertices:  tensor([0.5000, 0.5000, 0.5000])
faces:  tensor([0, 1, 2])
Vertices torch.Size([1, 4, 3]) torch.float32
Faces torch.Size([1, 4, 3]) torch.int64

texture_rgb = torch.ones_like(vertices) # N X 3
texture_rgb = texture_rgb * torch.tensor([0.7,0.7, 0])
textures = pytorch3d.renderer.TexturesVertex(texture_rgb)
print(texture_rgb)
print(texture_rgb.shape)

tensor([[[0.7000, 0.7000, 0.0000],
         [0.7000, 0.7000, 0.0000],
         [0.7000, 0.7000, 0.0000],
         [0.7000, 0.7000, 0.0000]]])
torch.Size([1, 4, 3])

meshes = pytorch3d.structures.Meshes(
    verts=vertices,
    faces=faces,
    textures=textures,
)
print("Using GPU:", torch.cuda.is_available())
if torch.cuda.is_available():
    device = torch.device("cuda:0")
else:
    device = torch.device("cpu")
meshes = meshes.to(device)

Using GPU: False

cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R=torch.eye(3).unsqueeze(0),
    T=torch.tensor([[0, 0, 3]]),
    fov=60,
)

image_size = 512

renderer = get_mesh_renderer(image_size=image_size, device=device)

lights = pytorch3d.renderer.PointLights(location=[[0, 0, -3]])
image = renderer(meshes, cameras=cameras, lights=lights)
plt.imshow(image[0].cpu().numpy())

<matplotlib.image.AxesImage at 0x305131d00>

plot_scene({
    "figure": {
        "Mesh": meshes,
        "Camera": cameras,
    }
})

num_views = 36
R, T = pytorch3d.renderer.look_at_view_transform(
    dist=10,
    elev=0,
    azim=np.linspace(0, 360, num_views, endpoint=False),
)
many_cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R=R,
    T=T,
    device=device
)
images = renderer(meshes.extend(num_views), cameras=many_cameras, lights=lights)

import imageio
images=(images.cpu().numpy()*255).astype(np.uint8)
duration=1000//15
imageio.mimsave('tetrahedron.gif',images,duration=duration,loop=0)

2.1 Constructing a Tetrahedron (5 points)

Constructing a Cube

We define a cube by taking 8 points whose centroid is at the origin (0,0,0). The cube's surface is represented by 12 triangular faces, with each of the cube's original 6 square faces subdivided into two triangles.

vertices = torch.tensor(
    [[1, 1, 1],
     [1, -1, 1],
     [-1, 1, 1],
     [-1, -1, 1],
     [1, 1, -1],
     [1, -1, -1],
     [-1, 1, -1],
     [-1, -1, -1]]
).to(dtype=torch.float32)

faces = torch.tensor(
    [[0, 1, 3],
     [0, 2, 3],
     [0, 4, 5],
     [0, 5, 1],
     [0, 4, 6],
     [0, 2, 6],
     [4, 6, 7],
     [4, 5, 7],
     [3, 6, 7],
     [2, 3, 6],
     [1, 5, 7],
     [1, 3, 7]],
    dtype=torch.int64
)
print("vertices: ", vertices[0])
print("faces: ", faces[0])
vertices = vertices.unsqueeze(0)  # 1 x N_v x 3
faces = faces.unsqueeze(0)
print("Vertices", vertices.shape,vertices.dtype)
print("Faces", faces.shape,faces.dtype)

vertices:  tensor([1., 1., 1.])
faces:  tensor([0, 1, 3])
Vertices torch.Size([1, 8, 3]) torch.float32
Faces torch.Size([1, 12, 3]) torch.int64

texture_rgb = torch.ones_like(vertices) # N X 3
texture_rgb = texture_rgb * torch.tensor([0.7,0.7, 0])
textures = pytorch3d.renderer.TexturesVertex(texture_rgb)
print(texture_rgb)
print(texture_rgb.shape)

tensor([[[0.7000, 0.7000, 0.0000],
         [0.7000, 0.7000, 0.0000],
         [0.7000, 0.7000, 0.0000],
         [0.7000, 0.7000, 0.0000],
         [0.7000, 0.7000, 0.0000],
         [0.7000, 0.7000, 0.0000],
         [0.7000, 0.7000, 0.0000],
         [0.7000, 0.7000, 0.0000]]])
torch.Size([1, 8, 3])

meshes = pytorch3d.structures.Meshes(
    verts=vertices, # batched tensor or a list of tensors
    faces=faces,
    textures=textures,
)

renderer = get_mesh_renderer(image_size=image_size, device=device)

cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R=torch.eye(3).unsqueeze(0),
    T=torch.tensor([[0, 0, 3]]),
    fov=60,
)
lights = pytorch3d.renderer.PointLights(location=[[3, 3, 3]])
image = renderer(meshes, cameras=cameras, lights=lights)
plt.imshow(image[0].cpu().numpy())

<matplotlib.image.AxesImage at 0x134ec1e50>

plot_scene({
    "figure": {
        "Mesh": meshes,
        "Camera": cameras,
    }
})

num_views = 36
R, T = pytorch3d.renderer.look_at_view_transform(
    dist=10,
    elev=0,
    azim=np.linspace(0, 360, num_views, endpoint=False),
)
many_cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R=R,
    T=T,
    device=device
)
images = renderer(meshes.extend(num_views), cameras=many_cameras, lights=lights)
images=(images.cpu().numpy()*255).astype(np.uint8)
duration=1000//15
imageio.mimsave('cube.gif',images,duration=duration,loop=0)

2.2 Constructing a Cube (5 points)

Re-Texturing a mesh

vertices, faces = load_cow_mesh(path="data/cow.obj")
vertices=vertices.unsqueeze(0)
faces=faces.unsqueeze(0)

colour1=torch.tensor([0,0,1]).unsqueeze(0).repeat(2930,1)
colour2=torch.tensor([1,0,0]).unsqueeze(0).repeat(2930,1)
"""texture_rgb = vertices.clone()
texture_rgb = (texture_rgb - texture_rgb.min()) / (texture_rgb.max() - texture_rgb.min())
texture_rgb /= texture_rgb.norm(dim=2, keepdim=True)
textures_rainbow = pytorch3d.renderer.TexturesVertex(texture_rgb.to(device))
meshes.textures = textures_rainbow"""
texture_gradiant=vertices.clone()
alpha=vertices[:,:,2] -vertices[:,:,2].min()/(vertices[:,:,2].max()-vertices[:,:,2].min())
alpha=alpha.squeeze(0).unsqueeze(1).repeat(1,3)

colour=colour1*alpha + (1-alpha)*colour2
textures_grad=colour.unsqueeze(0)
textures_grad=pytorch3d.renderer.TexturesVertex(textures_grad.to(device))

meshes = pytorch3d.structures.Meshes(
    verts=vertices, # batched tensor or a list of tensors
    faces=faces,
    textures=textures_grad,
)

import math
theta = math.radians(90)
R = torch.tensor([[math.cos(theta),0,math.sin(theta)],
                 [0,1,0],
                 [-math.sin(theta),0,math.cos(theta)]]).unsqueeze(0)
T = torch.tensor([[0.2, 0, 3]])

cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R=R,
    T=T,
    fov=60,
    device=device,
)
renderer = get_mesh_renderer(image_size=image_size)

image = renderer(meshes, cameras=cameras)
image = np.clip(image[0].cpu().numpy(),0,1)
plt.imshow(image)

<matplotlib.image.AxesImage at 0x131b67730>

#GIF Generation
num_views = 20
R, T = pytorch3d.renderer.look_at_view_transform(
    dist=10,
    elev=0,
    azim=np.linspace(0, 360, num_views, endpoint=False),
)
many_cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R=R,
    T=T,
    device=device
)
images = renderer(meshes.extend(num_views), cameras=many_cameras)
images=(images.cpu().numpy()*255).astype(np.uint8)
duration=1000//15
imageio.mimsave('re-texturing.gif',images,duration=duration,loop=True)

3. Re-texturing a mesh (10 points)

For color 1 I chose red(1,0,0) and for color 2 I chose blue(0,0,1)

4. Camera Transformations (10 points)

Transform 1: R_relative = [[math.cos(theta),math.sin(theta),0],[-math.sin(theta),math.cos(theta),0],[0,0,1]], T_relative = [0,0,0]

Transform 2: R_relative=[[1, 0, 0], [0, 1, 0], [0, 0, 1]], T_relative=[0, 0, 2]

Transform 3: R_relative = R_relative=[[1, 0, 0], [0, 1, 0], [0, 0, 1]], T_relative=[0.5, -0.5, 0]

Transform 4: R_relative =[[math.cos(theta),0,math.sin(theta)/3],[0,1,0],[-math.sin(theta),0,math.cos(theta)/3]], T_relative=[-1, 0, 3]

%run camera_transforms.py

Point Clouds

import starter.render_generic

data=starter.render_generic.load_rgbd_data()
img1=torch.tensor(data['rgb1'])
img2=torch.tensor(data['rgb2'])
depth1=torch.tensor(data['depth1'])
depth2=torch.tensor(data['depth2'])
mask1=torch.tensor(data['mask1'])
mask2=torch.tensor(data['mask2'])
camera1=data['cameras1']
camera2=data['cameras2']

plt.imshow(img1)

<matplotlib.image.AxesImage at 0x133e3b7f0>

pts1,rgb1=unproject_depth_image(img1, mask1, depth1, camera1)
pts2,rgb2=unproject_depth_image(img2, mask2, depth2, camera2)
pts1,rgb1=pts1.unsqueeze(0),rgb1.unsqueeze(0)
pts2,rgb2=pts2.unsqueeze(0),rgb2.unsqueeze(0)

/opt/anaconda3/envs/L3D/lib/python3.9/site-packages/torch/functional.py:554: UserWarning:

torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at /Users/runner/work/pytorch/pytorch/pytorch/aten/src/ATen/native/TensorShape.cpp:4324.)

point_cloud_1 = pytorch3d.structures.Pointclouds(
    points=pts1, features=rgb1
)
point_cloud_2 = pytorch3d.structures.Pointclouds(
    points=pts2, features=rgb2
)

cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R = torch.tensor([
    [-1,  0,  0],
    [ 0, -1,  0],
    [ 0,  0,  1]
]).unsqueeze(0),
    T=torch.tensor([[0, 0, 15]]),
    fov=60,
)

from starter.utils import get_points_renderer
points_renderer = get_points_renderer(
    image_size=256,
    radius=0.01,
)
rend_1 = points_renderer(point_cloud_1, cameras=cameras)
image_1 = rend_1[0, ..., :3].numpy()  # (B, H, W, 4) -> (H, W, 3).
rend_2 = points_renderer(point_cloud_2, cameras=cameras)
image_2 = rend_2[0, ..., :3].numpy()

plt.imshow(image_1)

<matplotlib.image.AxesImage at 0x133f672e0>

plt.imshow(image_2)

<matplotlib.image.AxesImage at 0x13402fbb0>

#point_cloud_1 and point_cloud_2
pts_union=torch.cat((pts1,pts2),dim=1)
rgb_union=torch.cat((rgb1,rgb2),dim=1)
point_cloud_3=pytorch3d.structures.Pointclouds(
    points=pts_union, features=rgb_union
)

#GIF Generation
num_views = 10
R, T = pytorch3d.renderer.look_at_view_transform(
    dist=10,
    elev=0,
    azim=np.linspace(0, 360, num_views, endpoint=False),
)
R1 = torch.tensor([[1, 0, 0], [0, -1, 0], [0, 0, 1]], device=R.device, dtype=R.dtype)

# Apply the flip to the original rotation matrix
R = R @ R1
many_cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R=R,
    T=T,
    device=device
)
points_renderer = get_points_renderer(
    image_size=256,
    radius=0.01,
)
images = points_renderer(point_cloud_1.extend(num_views), cameras=many_cameras)
images=(images.cpu().numpy()*255).astype(np.uint8)
duration=1000//15
imageio.mimsave('point_cloud_1.gif',images,duration=duration,loop=0)

images = points_renderer(point_cloud_2.extend(num_views), cameras=many_cameras)
images=(images.cpu().numpy()*255).astype(np.uint8)
duration=1000//15
imageio.mimsave('point_cloud_2.gif',images,duration=duration,loop=0)

images = points_renderer(point_cloud_3.extend(num_views), cameras=many_cameras)
images=(images.cpu().numpy()*255).astype(np.uint8)
duration=1000//15
imageio.mimsave('point_cloud_3.gif',images,duration=duration,loop=0)

5.1 Rendering Point Clouds from RGB-D Images (10 points)

First Image

Second Image

Union Image

Rendering a Torus

r=0.5
Rr=1.5
num_samples=400
phi = torch.linspace(0, 2 * np.pi, num_samples)
theta = torch.linspace(0, np.pi, num_samples)
Phi, Theta = torch.meshgrid(phi, theta)
#Parametric Function
x = (Rr+r*torch.sin(theta))*torch.cos(Phi)
y = r*torch.cos(Theta)
z = (Rr+r*torch.sin(theta)) * torch.sin(Phi)

pts=torch.stack((x.flatten(),y.flatten(),z.flatten()),dim=1)
rgb=(pts-pts.min())/(pts.max()-pts.min())
pts=pts.unsqueeze(0)
rgb=rgb.unsqueeze(0)
pts.shape,rgb.shape

(torch.Size([1, 160000, 3]), torch.Size([1, 160000, 3]))

angle = math.pi / 2
"""R = torch.tensor([
    [math.cos(angle), 0, math.sin(angle)],
    [0, 1, 0],
    [-math.sin(angle), 0, math.cos(angle)]
]).unsqueeze(0)"""
R, T = pytorch3d.renderer.look_at_view_transform(dist=8, elev=45, azim=45)
torus_pc=pytorch3d.structures.Pointclouds(points=pts,features=rgb)
cameras = pytorch3d.renderer.FoVPerspectiveCameras(R=R,T=T, device=device)
renderer = get_points_renderer(image_size=image_size, device=device)
rend = renderer(torus_pc, cameras=cameras)
plt.imshow(rend[0, ..., :3].cpu().numpy())

<matplotlib.image.AxesImage at 0x131b0ca00>

plot_scene({
    "figure": {
        "Pointcloud": torus_pc,
        "Camera": cameras,
    }
})

5.2 Parametric Functions (10 + 5 points)

Torus

Shape of my choice: Moebius

num_samples=400
u = torch.linspace(0, 2 * np.pi, num_samples)
v = torch.linspace(-1, 1, num_samples)
u,v = torch.meshgrid(u, v)
#Parametric Function
x = (1+(v/2)*torch.cos(u/2))*torch.cos(u)
y = (1+(v/2)*torch.cos(u/2))*torch.sin(u)
z = (v/2)*torch.sin(u/2)

pts=torch.stack((x.flatten(),y.flatten(),z.flatten()),dim=1)
rgb=(pts-pts.min())/(pts.max()-pts.min())
pts=pts.unsqueeze(0)
rgb=rgb.unsqueeze(0)
pts.shape,rgb.shape

(torch.Size([1, 160000, 3]), torch.Size([1, 160000, 3]))

R, T = pytorch3d.renderer.look_at_view_transform(dist=15, elev=45, azim=45)
moebius_pc=pytorch3d.structures.Pointclouds(points=pts,features=rgb)
cameras = pytorch3d.renderer.FoVPerspectiveCameras(R=R,T=T, device=device)
renderer = get_points_renderer(image_size=image_size, device=device)
rend = renderer(moebius_pc, cameras=cameras)
plt.imshow(rend[0, ..., :3].cpu().numpy())

<matplotlib.image.AxesImage at 0x131c3d7c0>

plot_scene({
    "figure": {
        "Pointcloud": moebius_pc,
        "Camera": cameras,
    }
})

num_views = 10
R, T = pytorch3d.renderer.look_at_view_transform(
    dist=10,
    elev=45,
    azim=np.linspace(0, 360, num_views, endpoint=False),
)
many_cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R=R,
    T=T,
    device=device
)
points_renderer = get_points_renderer(
    image_size=256,
    radius=0.01,
)
images = points_renderer(torus_pc.extend(num_views), cameras=many_cameras)
images=(images.cpu().numpy()*255).astype(np.uint8)
duration=1000//15
imageio.mimsave('torus.gif',images,duration=duration,loop=0)

images = points_renderer(moebius_pc.extend(num_views), cameras=many_cameras)
images=(images.cpu().numpy()*255).astype(np.uint8)
duration=1000//15
imageio.mimsave('moebius.gif',images,duration=duration,loop=0)

Moebius (5 points)

Implicit Functions

import mcubes

r=0.5
R=1.5
min_val=-5
max_val=5
voxel_size=64

X,Y,Z=torch.meshgrid([torch.linspace(min_val,max_val,voxel_size)]*3)
fn= (R - torch.sqrt(X**2 + Y**2))**2 + Z**2 - r**2
vertices, faces = mcubes.marching_cubes(mcubes.smooth(fn), isovalue=0)
vertices = torch.tensor(vertices).float()
faces = torch.tensor(faces.astype(int))
vertices = (vertices / voxel_size) * (max_val - min_val) + min_val
textures = (vertices - vertices.min()) / (vertices.max() - vertices.min())
textures = pytorch3d.renderer.TexturesVertex(vertices.unsqueeze(0))

mesh = pytorch3d.structures.Meshes([vertices], [faces], textures=textures).to(
    device
)
lights = pytorch3d.renderer.PointLights(location=[[0, 0.0, -4.0]], device=device,)
renderer = get_mesh_renderer(image_size=image_size, device=device)
R, T = pytorch3d.renderer.look_at_view_transform(dist=8, elev=45, azim=45)
cameras = pytorch3d.renderer.FoVPerspectiveCameras(R=R, T=T, device=device)
rend = renderer(mesh, cameras=cameras, lights=lights)
plt.imshow(rend[0, ..., :3].detach().cpu().numpy().clip(0, 1))

<matplotlib.image.AxesImage at 0x1314f0070>

plot_scene({
    "figure": {
        "Mesh": mesh,
        "Camera": cameras,
    }
})

num_views = 10
R, T = pytorch3d.renderer.look_at_view_transform(
    dist=10,
    elev=45,
    azim=np.linspace(0, 360, num_views, endpoint=False),
)
many_cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R=R,
    T=T,
    device=device
)
images = renderer(mesh.extend(num_views), cameras=many_cameras)
images=(images.cpu().numpy()*255).astype(np.uint8)
duration=1000//15
imageio.mimsave('torus_implicit.gif',images,duration=duration,loop=0)

5.3 Implicit Surfaces (15 + 5 points)

Torus Mesh

Tradeoff between mesh vs point cloud (5 points)

1. The rendering quality of meshes are better than Point Clouds as meshes are smooth, continuous and water tight whereas Point Clouds are sampled points making it discrete.

2. The rendering speed of Point Clouds are faster as they dont need triangular rasterization unlike Meshes.

3. Point Clouds are easier to use as you only need a list of points and dont need the connectivity between points to form faces.

4. Point Clouds require less memory in most cases as meshes store both vertices and faces.

My choice of Implicit Function: Helicoid

min_val=-5
max_val=5
voxel_size=64

X,Y,Z=torch.meshgrid([torch.linspace(min_val,max_val,voxel_size)]*3)
fn= X*torch.sin(Z) - Y*torch.cos(Z)
vertices, faces = mcubes.marching_cubes(mcubes.smooth(fn), isovalue=0)
vertices = torch.tensor(vertices).float()
faces = torch.tensor(faces.astype(int))
vertices = (vertices / voxel_size) * (max_val - min_val) + min_val
textures = (vertices - vertices.min()) / (vertices.max() - vertices.min())
textures = pytorch3d.renderer.TexturesVertex(vertices.unsqueeze(0))

mesh = pytorch3d.structures.Meshes([vertices], [faces], textures=textures).to(
    device
)
lights = pytorch3d.renderer.PointLights(location=[[0, 0.0, -4.0]], device=device,)
renderer = get_mesh_renderer(image_size=image_size, device=device)
R, T = pytorch3d.renderer.look_at_view_transform(dist=20, elev=45, azim=45)
cameras = pytorch3d.renderer.FoVPerspectiveCameras(R=R, T=T, device=device)
rend = renderer(mesh, cameras=cameras, lights=lights)
plt.imshow(rend[0, ..., :3].detach().cpu().numpy().clip(0, 1))

<matplotlib.image.AxesImage at 0x131a5a4f0>

plot_scene({
    "figure": {
        "Mesh": mesh,
        "Camera": cameras,
    }
})

num_views = 10
R, T = pytorch3d.renderer.look_at_view_transform(
    dist=20,
    elev=45,
    azim=np.linspace(0, 360, num_views, endpoint=False),
)
many_cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R=R,
    T=T,
    device=device
)
images = renderer(mesh.extend(num_views), cameras=many_cameras)
images=(images.cpu().numpy()*255).astype(np.uint8)
duration=1000//15
imageio.mimsave('helicoid_im.gif',images,duration=duration,loop=0)

Helicoid (5 points)

Something Fun from Free3D (10 points)

Shape Link : https://free3d.com/3d-model/ferrari-formula-1-72527.html

from pytorch3d.io import load_obj
verts,faces,aux =load_obj("data/f1/F1.obj", device=device, load_textures=True, create_texture_atlas=True)

from pytorch3d.structures import Meshes
from pytorch3d.renderer import TexturesAtlas
atlas = TexturesAtlas(atlas=[aux.texture_atlas])
car_mesh = Meshes(
    verts=[verts],
    faces=[faces.verts_idx],
    textures=atlas
)

from pytorch3d.renderer import BlendParams

R, T = pytorch3d.renderer.look_at_view_transform(
    dist=350,
    elev=20,
    azim=-65,
)
print(R,T)
cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R=R,
    T=T,
    fov=60,
    device=device,)

image_size=512
renderer = get_mesh_renderer(image_size=image_size, device=device)

image = renderer(car_mesh, cameras=cameras)
plt.imshow(image[0].cpu().numpy())

tensor([[[-0.4226,  0.3100,  0.8517],
         [ 0.0000,  0.9397, -0.3420],
         [-0.9063, -0.1445, -0.3971]]]) tensor([[-6.0759e-06, -2.7784e-06,  3.5000e+02]])

<matplotlib.image.AxesImage at 0x130119d00>

plot_scene({
    "figure": {
        "Mesh": car_mesh,
        "Camera": cameras,
    }
})

num_views = 30
R, T = pytorch3d.renderer.look_at_view_transform(
    dist=400,
    elev=np.linspace(0, 180, num_views, endpoint=False),
    azim=-65,
)
many_cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R=R,
    T=T,
    device=device
)
images = renderer(car_mesh.extend(num_views), cameras=many_cameras)
images=(images.cpu().numpy()*255).astype(np.uint8)
duration=1000//15

#Used PIL instead of imageio to avoid image superimposition of previous frames
from PIL import Image
frames = [Image.fromarray(img) for img in images]
frames[0].save(
    "f1_car2.gif",
    save_all=True,
    append_images=frames[1:],
    duration=duration,
    loop=0,
    disposal=2
)

Goofy Gif

Extra Credits

vertices, faces = load_cow_mesh(path="data/cow.obj")
vertices.shape,faces.shape

(torch.Size([2930, 3]), torch.Size([5856, 3]))

sample_pts=1000
vertex1=vertices[faces[:,0]]
vertex2=vertices[faces[:,1]]
vertex3=vertices[faces[:,2]]

area=0.5*torch.linalg.norm(torch.cross(vertex2-vertex1,vertex3-vertex1),dim=1)
vertex1.shape,vertex2.shape,vertex3.shape,

(torch.Size([5856, 3]), torch.Size([5856, 3]), torch.Size([5856, 3]))

norm=area/area.sum()
ind=torch.multinomial(norm,sample_pts,replacement=True)
ind.shape

torch.Size([1000])

vertex1=vertex1[ind]
vertex2=vertex2[ind]
vertex3=vertex3[ind]

u = torch.rand(sample_pts, 1)
v = torch.rand(sample_pts, 1)
mask = (u + v > 1)
u[mask] = 1 - u[mask]
v[mask] = 1 - v[mask]
w = 1 - (u + v)
samples = u * vertex1 + v * vertex2 + w * vertex3
samples.shape

torch.Size([1000, 3])

num_views = 10
R, T = pytorch3d.renderer.look_at_view_transform(
    dist=3,
    elev=0,
    azim=np.linspace(0, 360, num_views, endpoint=False),
)
many_cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R=R,
    T=T,
    device=device
)
points_renderer = get_points_renderer(
    image_size=256,
    radius=0.01,
)
point_cloud = pytorch3d.structures.Pointclouds(
    points=[samples],
    features=[torch.ones_like(samples)]
)
images = points_renderer(point_cloud.extend(num_views), cameras=many_cameras)
images=(images.cpu().numpy()*255).astype(np.uint8)
duration=1000//15
imageio.mimsave('sampling_pc.gif',images,duration=duration,loop=0)

num_views = 36
R, T = pytorch3d.renderer.look_at_view_transform(
    dist=3,
    elev=0,
    azim=np.linspace(0, 360, num_views, endpoint=False),
)
many_cameras = pytorch3d.renderer.FoVPerspectiveCameras(
    R=R,
    T=T,
    device=device
)
vertices=vertices.unsqueeze(0)
faces=faces.unsqueeze(0)
texture_rgb = torch.ones_like(vertices) # N X 3
texture_rgb = texture_rgb * torch.tensor([0.7,0.7, 0])
textures = pytorch3d.renderer.TexturesVertex(texture_rgb)
meshes = pytorch3d.structures.Meshes(
    verts=vertices,
    faces=faces,
    textures=textures,
)
images = renderer(meshes.extend(num_views), cameras=many_cameras, lights=lights)
images=(images.cpu().numpy()*255).astype(np.uint8)
duration=1000//15
imageio.mimsave('sampling_mesh.gif',images,duration=duration,loop=0)

(Extra Credit) 7. Sampling Points on Meshes (10 points)

Point Cloud vs Mesh