from matplotlib import pyplot as plt
import numpy as np
import torch
import numpy as np
from typing import List
import sys
sys.path.append('./submodules/gaussian-splatting/')
from scene.cameras import Camera
from PIL import Image
import imageio
from scipy.interpolate import splprep, splev
import cv2
import numpy as np
import plotly.graph_objects as go
import numpy as np
from scipy.spatial.transform import Rotation as R, Slerp
from scipy.spatial import distance_matrix
from sklearn.decomposition import PCA
from scipy.interpolate import splprep, splev
from typing import List
from sklearn.mixture import GaussianMixture
def render_gaussians_rgb(generator3DGS, viewpoint_cam, visualize=False):
"""
Simply render gaussians from the generator3DGS from the viewpoint_cam.
Args:
generator3DGS : instance of the Generator3DGS class from the networks.py file
viewpoint_cam : camera instance
visualize : boolean flag. If True, will call pyplot function and render image inplace
Returns:
uint8 numpy array with shape (H, W, 3) representing the image
"""
with torch.no_grad():
render_pkg = generator3DGS(viewpoint_cam)
image = render_pkg["render"]
image_np = image.clone().detach().cpu().numpy().transpose(1, 2, 0)
# Clip values to be in the range [0, 1]
image_np = np.clip(image_np * 255, 0, 255).astype(np.uint8)
if visualize:
plt.figure(figsize=(12, 8))
plt.imshow(image_np)
plt.show()
return image_np
def render_gaussians_D_scores(generator3DGS, viewpoint_cam, mask=None, mask_channel=0, visualize=False):
"""
Simply render D_scores of gaussians from the generator3DGS from the viewpoint_cam.
Args:
generator3DGS : instance of the Generator3DGS class from the networks.py file
viewpoint_cam : camera instance
visualize : boolean flag. If True, will call pyplot function and render image inplace
mask : optional mask to highlight specific gaussians. Must be of shape (N) where N is the numnber
of gaussians in generator3DGS.gaussians. Must be a torch tensor of floats, please scale according
to how much color you want to have. Recommended mask value is 10.
mask_channel: to which color channel should we add mask
Returns:
uint8 numpy array with shape (H, W, 3) representing the generator3DGS.gaussians.D_scores rendered as colors
"""
with torch.no_grad():
# Visualize D_scores
generator3DGS.gaussians._features_dc = generator3DGS.gaussians._features_dc * 1e-4 + \
torch.stack([generator3DGS.gaussians.D_scores] * 3, axis=-1)
generator3DGS.gaussians._features_rest = generator3DGS.gaussians._features_rest * 1e-4
if mask is not None:
generator3DGS.gaussians._features_dc[..., mask_channel] += mask.unsqueeze(-1)
render_pkg = generator3DGS(viewpoint_cam)
image = render_pkg["render"]
image_np = image.clone().detach().cpu().numpy().transpose(1, 2, 0)
# Clip values to be in the range [0, 1]
image_np = np.clip(image_np * 255, 0, 255).astype(np.uint8)
if visualize:
plt.figure(figsize=(12, 8))
plt.imshow(image_np)
plt.show()
if mask is not None:
generator3DGS.gaussians._features_dc[..., mask_channel] -= mask.unsqueeze(-1)
generator3DGS.gaussians._features_dc = (generator3DGS.gaussians._features_dc - \
torch.stack([generator3DGS.gaussians.D_scores] * 3, axis=-1)) * 1e4
generator3DGS.gaussians._features_rest = generator3DGS.gaussians._features_rest * 1e4
return image_np
def normalize(v):
"""
Normalize a vector to unit length.
Parameters:
v (np.ndarray): Input vector.
Returns:
np.ndarray: Unit vector in the same direction as `v`.
"""
return v / np.linalg.norm(v)
def look_at_rotation(camera_position: np.ndarray, target: np.ndarray, world_up=np.array([0, 1, 0])):
"""
Compute a rotation matrix for a camera looking at a target point.
Parameters:
camera_position (np.ndarray): The 3D position of the camera.
target (np.ndarray): The point the camera should look at.
world_up (np.ndarray): A vector that defines the global 'up' direction.
Returns:
np.ndarray: A 3x3 rotation matrix (camera-to-world) with columns [right, up, forward].
"""
z_axis = normalize(target - camera_position) # Forward direction
x_axis = normalize(np.cross(world_up, z_axis)) # Right direction
y_axis = np.cross(z_axis, x_axis) # Recomputed up
return np.stack([x_axis, y_axis, z_axis], axis=1)
def generate_circular_camera_path(existing_cameras: List[Camera], N: int = 12, radius_scale: float = 1.0, d: float = 2.0) -> List[Camera]:
"""
Generate a circular path of cameras around an existing camera group,
with each new camera oriented to look at the average viewing direction.
Parameters:
existing_cameras (List[Camera]): List of existing camera objects to estimate average orientation and layout.
N (int): Number of new cameras to generate along the circular path.
radius_scale (float): Scale factor to adjust the radius of the circle.
d (float): Distance ahead of each camera used to estimate its look-at point.
Returns:
List[Camera]: A list of newly generated Camera objects forming a circular path and oriented toward a shared view center.
"""
# Step 1: Compute average camera position
center = np.mean([cam.T for cam in existing_cameras], axis=0)
# Estimate where each camera is looking
# d denotes how far ahead each camera sees — you can scale this
look_targets = [cam.T + cam.R[:, 2] * d for cam in existing_cameras]
center_of_view = np.mean(look_targets, axis=0)
# Step 2: Define circular plane basis using fixed up vector
avg_forward = normalize(np.mean([cam.R[:, 2] for cam in existing_cameras], axis=0))
up_guess = np.array([0, 1, 0])
right = normalize(np.cross(avg_forward, up_guess))
up = normalize(np.cross(right, avg_forward))
# Step 3: Estimate radius
avg_radius = np.mean([np.linalg.norm(cam.T - center) for cam in existing_cameras]) * radius_scale
# Step 4: Create cameras on a circular path
angles = np.linspace(0, 2 * np.pi, N, endpoint=False)
reference_cam = existing_cameras[0]
new_cameras = []
for i, a in enumerate(angles):
position = center + avg_radius * (np.cos(a) * right + np.sin(a) * up)
if d < 1e-5 or radius_scale < 1e-5:
# Use same orientation as the first camera
R = reference_cam.R.copy()
else:
# Change orientation
R = look_at_rotation(position, center_of_view)
new_cameras.append(Camera(
R=R,
T=position, # New position
FoVx=reference_cam.FoVx,
FoVy=reference_cam.FoVy,
resolution=(reference_cam.image_width, reference_cam.image_height),
colmap_id=-1,
depth_params=None,
image=Image.fromarray(np.zeros((reference_cam.image_height, reference_cam.image_width, 3), dtype=np.uint8)),
invdepthmap=None,
image_name=f"circular_a={a:.3f}",
uid=i
))
return new_cameras
def save_numpy_frames_as_gif(frames, output_path="animation.gif", duration=100):
"""
Save a list of RGB NumPy frames as a looping GIF animation.
Parameters:
frames (List[np.ndarray]): List of RGB images as uint8 NumPy arrays (shape HxWx3).
output_path (str): Path to save the output GIF.
duration (int): Duration per frame in milliseconds.
Returns:
None
"""
pil_frames = [Image.fromarray(f) for f in frames]
pil_frames[0].save(
output_path,
save_all=True,
append_images=pil_frames[1:],
duration=duration, # duration per frame in ms
loop=0
)
print(f"GIF saved to: {output_path}")
def center_crop_frame(frame: np.ndarray, crop_fraction: float) -> np.ndarray:
"""
Crop the central region of the frame by the given fraction.
Parameters:
frame (np.ndarray): Input RGB image (H, W, 3).
crop_fraction (float): Fraction of the original size to retain (e.g., 0.8 keeps 80%).
Returns:
np.ndarray: Cropped RGB image.
"""
if crop_fraction >= 1.0:
return frame
h, w, _ = frame.shape
new_h, new_w = int(h * crop_fraction), int(w * crop_fraction)
start_y = (h - new_h) // 2
start_x = (w - new_w) // 2
return frame[start_y:start_y + new_h, start_x:start_x + new_w, :]
def generate_smooth_closed_camera_path(existing_cameras: List[Camera], N: int = 120, d: float = 2.0, s=.25) -> List[Camera]:
"""
Generate a smooth, closed path interpolating the positions of existing cameras.
Parameters:
existing_cameras (List[Camera]): List of existing cameras.
N (int): Number of points (cameras) to sample along the smooth path.
d (float): Distance ahead for estimating the center of view.
Returns:
List[Camera]: A list of smoothly moving Camera objects along a closed loop.
"""
# Step 1: Extract camera positions
positions = np.array([cam.T for cam in existing_cameras])
# Step 2: Estimate center of view
look_targets = [cam.T + cam.R[:, 2] * d for cam in existing_cameras]
center_of_view = np.mean(look_targets, axis=0)
# Step 3: Fit a smooth closed spline through the positions
positions = np.vstack([positions, positions[0]]) # close the loop
tck, u = splprep(positions.T, s=s, per=True) # periodic=True for closed loop
# Step 4: Sample points along the spline
u_fine = np.linspace(0, 1, N)
smooth_path = np.stack(splev(u_fine, tck), axis=-1)
# Step 5: Generate cameras along the smooth path
reference_cam = existing_cameras[0]
new_cameras = []
for i, pos in enumerate(smooth_path):
R = look_at_rotation(pos, center_of_view)
new_cameras.append(Camera(
R=R,
T=pos,
FoVx=reference_cam.FoVx,
FoVy=reference_cam.FoVy,
resolution=(reference_cam.image_width, reference_cam.image_height),
colmap_id=-1,
depth_params=None,
image=Image.fromarray(np.zeros((reference_cam.image_height, reference_cam.image_width, 3), dtype=np.uint8)),
invdepthmap=None,
image_name=f"smooth_path_i={i}",
uid=i
))
return new_cameras
def save_numpy_frames_as_mp4(frames, output_path="animation.mp4", fps=10, center_crop: float = 1.0):
"""
Save a list of RGB NumPy frames as an MP4 video with optional center cropping.
Parameters:
frames (List[np.ndarray]): List of RGB images as uint8 NumPy arrays (shape HxWx3).
output_path (str): Path to save the output MP4.
fps (int): Frames per second for playback speed.
center_crop (float): Fraction (0 < center_crop <= 1.0) of central region to retain.
Use 1.0 for no cropping; 0.8 to crop to 80% center region.
Returns:
None
"""
with imageio.get_writer(output_path, fps=fps, codec='libx264', quality=8) as writer:
for frame in frames:
cropped = center_crop_frame(frame, center_crop)
writer.append_data(cropped)
print(f"MP4 saved to: {output_path}")
def put_text_on_image(img: np.ndarray, text: str) -> np.ndarray:
"""
Draws multiline white text on a copy of the input image, positioned near the bottom
and around 80% of the image width. Handles '\n' characters to split text into multiple lines.
Args:
img (np.ndarray): Input image as a (H, W, 3) uint8 numpy array.
text (str): Text string to draw on the image. Newlines '\n' are treated as line breaks.
Returns:
np.ndarray: The output image with the text drawn on it.
Notes:
- The function automatically adjusts line spacing and prevents text from going outside the image.
- Text is drawn in white with small font size (0.5) for minimal visual impact.
"""
img = img.copy()
height, width, _ = img.shape
font = cv2.FONT_HERSHEY_SIMPLEX
font_scale = 1.
color = (255, 255, 255)
thickness = 2
line_spacing = 5 # extra pixels between lines
lines = text.split('\n')
# Precompute the maximum text width to adjust starting x
max_text_width = max(cv2.getTextSize(line, font, font_scale, thickness)[0][0] for line in lines)
x = int(0.8 * width)
x = min(x, width - max_text_width - 30) # margin on right
#x = int(0.03 * width)
# Start near the bottom, but move up depending on number of lines
total_text_height = len(lines) * (cv2.getTextSize('A', font, font_scale, thickness)[0][1] + line_spacing)
y_start = int(height*0.9) - total_text_height # 30 pixels from bottom
for i, line in enumerate(lines):
y = y_start + i * (cv2.getTextSize(line, font, font_scale, thickness)[0][1] + line_spacing)
cv2.putText(img, line, (x, y), font, font_scale, color, thickness, cv2.LINE_AA)
return img
def catmull_rom_spline(P0, P1, P2, P3, n_points=20):
"""
Compute Catmull-Rom spline segment between P1 and P2.
"""
t = np.linspace(0, 1, n_points)[:, None]
M = 0.5 * np.array([
[-1, 3, -3, 1],
[ 2, -5, 4, -1],
[-1, 0, 1, 0],
[ 0, 2, 0, 0]
])
G = np.stack([P0, P1, P2, P3], axis=0)
T = np.concatenate([t**3, t**2, t, np.ones_like(t)], axis=1)
return T @ M @ G
def sort_cameras_pca(existing_cameras: List[Camera]):
"""
Sort cameras along the main PCA axis.
"""
positions = np.array([cam.T for cam in existing_cameras])
pca = PCA(n_components=1)
scores = pca.fit_transform(positions)
sorted_indices = np.argsort(scores[:, 0])
return sorted_indices
def generate_fully_smooth_cameras(existing_cameras: List[Camera],
n_selected: int = 30,
n_points_per_segment: int = 20,
d: float = 2.0,
closed: bool = False) -> List[Camera]:
"""
Generate a fully smooth camera path using PCA ordering, global Catmull-Rom spline for positions, and global SLERP for orientations.
Args:
existing_cameras (List[Camera]): List of input cameras.
n_selected (int): Number of cameras to select after sorting.
n_points_per_segment (int): Number of interpolated points per spline segment.
d (float): Distance ahead for estimating center of view.
closed (bool): Whether to close the path.
Returns:
List[Camera]: List of smoothly moving Camera objects.
"""
# 1. Sort cameras along PCA axis
sorted_indices = sort_cameras_pca(existing_cameras)
sorted_cameras = [existing_cameras[i] for i in sorted_indices]
positions = np.array([cam.T for cam in sorted_cameras])
# 2. Subsample uniformly
idx = np.linspace(0, len(positions) - 1, n_selected).astype(int)
sampled_positions = positions[idx]
sampled_cameras = [sorted_cameras[i] for i in idx]
# 3. Prepare for Catmull-Rom
if closed:
sampled_positions = np.vstack([sampled_positions[-1], sampled_positions, sampled_positions[0], sampled_positions[1]])
else:
sampled_positions = np.vstack([sampled_positions[0], sampled_positions, sampled_positions[-1], sampled_positions[-1]])
# 4. Generate smooth path positions
path_positions = []
for i in range(1, len(sampled_positions) - 2):
segment = catmull_rom_spline(sampled_positions[i-1], sampled_positions[i], sampled_positions[i+1], sampled_positions[i+2], n_points_per_segment)
path_positions.append(segment)
path_positions = np.concatenate(path_positions, axis=0)
# 5. Global SLERP for rotations
rotations = R.from_matrix([cam.R for cam in sampled_cameras])
key_times = np.linspace(0, 1, len(rotations))
slerp = Slerp(key_times, rotations)
query_times = np.linspace(0, 1, len(path_positions))
interpolated_rotations = slerp(query_times)
# 6. Generate Camera objects
reference_cam = existing_cameras[0]
smooth_cameras = []
for i, pos in enumerate(path_positions):
R_interp = interpolated_rotations[i].as_matrix()
smooth_cameras.append(Camera(
R=R_interp,
T=pos,
FoVx=reference_cam.FoVx,
FoVy=reference_cam.FoVy,
resolution=(reference_cam.image_width, reference_cam.image_height),
colmap_id=-1,
depth_params=None,
image=Image.fromarray(np.zeros((reference_cam.image_height, reference_cam.image_width, 3), dtype=np.uint8)),
invdepthmap=None,
image_name=f"fully_smooth_path_i={i}",
uid=i
))
return smooth_cameras
def plot_cameras_and_smooth_path_with_orientation(existing_cameras: List[Camera], smooth_cameras: List[Camera], scale: float = 0.1):
"""
Plot input cameras and smooth path cameras with their orientations in 3D.
Args:
existing_cameras (List[Camera]): List of original input cameras.
smooth_cameras (List[Camera]): List of smooth path cameras.
scale (float): Length of orientation arrows.
Returns:
None
"""
# Input cameras
input_positions = np.array([cam.T for cam in existing_cameras])
# Smooth cameras
smooth_positions = np.array([cam.T for cam in smooth_cameras])
fig = go.Figure()
# Plot input camera positions
fig.add_trace(go.Scatter3d(
x=input_positions[:, 0], y=input_positions[:, 1], z=input_positions[:, 2],
mode='markers',
marker=dict(size=4, color='blue'),
name='Input Cameras'
))
# Plot smooth path positions
fig.add_trace(go.Scatter3d(
x=smooth_positions[:, 0], y=smooth_positions[:, 1], z=smooth_positions[:, 2],
mode='lines+markers',
line=dict(color='red', width=3),
marker=dict(size=2, color='red'),
name='Smooth Path Cameras'
))
# Plot input camera orientations
for cam in existing_cameras:
origin = cam.T
forward = cam.R[:, 2] # Forward direction
fig.add_trace(go.Cone(
x=[origin[0]], y=[origin[1]], z=[origin[2]],
u=[forward[0]], v=[forward[1]], w=[forward[2]],
colorscale=[[0, 'blue'], [1, 'blue']],
sizemode="absolute",
sizeref=scale,
anchor="tail",
showscale=False,
name='Input Camera Direction'
))
# Plot smooth camera orientations
for cam in smooth_cameras:
origin = cam.T
forward = cam.R[:, 2] # Forward direction
fig.add_trace(go.Cone(
x=[origin[0]], y=[origin[1]], z=[origin[2]],
u=[forward[0]], v=[forward[1]], w=[forward[2]],
colorscale=[[0, 'red'], [1, 'red']],
sizemode="absolute",
sizeref=scale,
anchor="tail",
showscale=False,
name='Smooth Camera Direction'
))
fig.update_layout(
scene=dict(
xaxis_title='X',
yaxis_title='Y',
zaxis_title='Z',
aspectmode='data'
),
title="Input Cameras and Smooth Path with Orientations",
margin=dict(l=0, r=0, b=0, t=30)
)
fig.show()
def solve_tsp_nearest_neighbor(points: np.ndarray):
"""
Solve TSP approximately using nearest neighbor heuristic.
Args:
points (np.ndarray): (N, 3) array of points.
Returns:
List[int]: Optimal visiting order of points.
"""
N = points.shape[0]
dist = distance_matrix(points, points)
visited = [0]
unvisited = set(range(1, N))
while unvisited:
last = visited[-1]
next_city = min(unvisited, key=lambda city: dist[last, city])
visited.append(next_city)
unvisited.remove(next_city)
return visited
def solve_tsp_2opt(points: np.ndarray, n_iter: int = 1000) -> np.ndarray:
"""
Solve TSP approximately using Nearest Neighbor + 2-Opt.
Args:
points (np.ndarray): Array of shape (N, D) with points.
n_iter (int): Number of 2-opt iterations.
Returns:
np.ndarray: Ordered list of indices.
"""
n_points = points.shape[0]
# === 1. Start with Nearest Neighbor
unvisited = list(range(n_points))
current = unvisited.pop(0)
path = [current]
while unvisited:
dists = np.linalg.norm(points[unvisited] - points[current], axis=1)
next_idx = unvisited[np.argmin(dists)]
unvisited.remove(next_idx)
path.append(next_idx)
current = next_idx
# === 2. Apply 2-Opt improvements
def path_length(path):
return np.sum(np.linalg.norm(points[path[i]] - points[path[i+1]], axis=0) for i in range(len(path)-1))
best_length = path_length(path)
improved = True
for _ in range(n_iter):
if not improved:
break
improved = False
for i in range(1, n_points - 2):
for j in range(i + 1, n_points):
if j - i == 1: continue
new_path = path[:i] + path[i:j][::-1] + path[j:]
new_length = path_length(new_path)
if new_length < best_length:
path = new_path
best_length = new_length
improved = True
break
if improved:
break
return np.array(path)
def generate_fully_smooth_cameras_with_tsp(existing_cameras: List[Camera],
n_selected: int = 30,
n_points_per_segment: int = 20,
d: float = 2.0,
closed: bool = False) -> List[Camera]:
"""
Generate a fully smooth camera path using TSP ordering, global Catmull-Rom spline for positions, and global SLERP for orientations.
Args:
existing_cameras (List[Camera]): List of input cameras.
n_selected (int): Number of cameras to select after ordering.
n_points_per_segment (int): Number of interpolated points per spline segment.
d (float): Distance ahead for estimating center of view.
closed (bool): Whether to close the path.
Returns:
List[Camera]: List of smoothly moving Camera objects.
"""
positions = np.array([cam.T for cam in existing_cameras])
# 1. Solve approximate TSP
order = solve_tsp_nearest_neighbor(positions)
ordered_cameras = [existing_cameras[i] for i in order]
ordered_positions = positions[order]
# 2. Subsample uniformly
idx = np.linspace(0, len(ordered_positions) - 1, n_selected).astype(int)
sampled_positions = ordered_positions[idx]
sampled_cameras = [ordered_cameras[i] for i in idx]
# 3. Prepare for Catmull-Rom
if closed:
sampled_positions = np.vstack([sampled_positions[-1], sampled_positions, sampled_positions[0], sampled_positions[1]])
else:
sampled_positions = np.vstack([sampled_positions[0], sampled_positions, sampled_positions[-1], sampled_positions[-1]])
# 4. Generate smooth path positions
path_positions = []
for i in range(1, len(sampled_positions) - 2):
segment = catmull_rom_spline(sampled_positions[i-1], sampled_positions[i], sampled_positions[i+1], sampled_positions[i+2], n_points_per_segment)
path_positions.append(segment)
path_positions = np.concatenate(path_positions, axis=0)
# 5. Global SLERP for rotations
rotations = R.from_matrix([cam.R for cam in sampled_cameras])
key_times = np.linspace(0, 1, len(rotations))
slerp = Slerp(key_times, rotations)
query_times = np.linspace(0, 1, len(path_positions))
interpolated_rotations = slerp(query_times)
# 6. Generate Camera objects
reference_cam = existing_cameras[0]
smooth_cameras = []
for i, pos in enumerate(path_positions):
R_interp = interpolated_rotations[i].as_matrix()
smooth_cameras.append(Camera(
R=R_interp,
T=pos,
FoVx=reference_cam.FoVx,
FoVy=reference_cam.FoVy,
resolution=(reference_cam.image_width, reference_cam.image_height),
colmap_id=-1,
depth_params=None,
image=Image.fromarray(np.zeros((reference_cam.image_height, reference_cam.image_width, 3), dtype=np.uint8)),
invdepthmap=None,
image_name=f"fully_smooth_path_i={i}",
uid=i
))
return smooth_cameras
from typing import List
import numpy as np
from sklearn.mixture import GaussianMixture
from scipy.spatial.transform import Rotation as R, Slerp
from PIL import Image
def generate_clustered_smooth_cameras_with_tsp(existing_cameras: List[Camera],
n_selected: int = 30,
n_points_per_segment: int = 20,
d: float = 2.0,
n_clusters: int = 5,
closed: bool = False) -> List[Camera]:
"""
Generate a fully smooth camera path using clustering + TSP between nearest cluster centers + TSP inside clusters.
Positions are normalized before clustering and denormalized before generating final cameras.
Args:
existing_cameras (List[Camera]): List of input cameras.
n_selected (int): Number of cameras to select after ordering.
n_points_per_segment (int): Number of interpolated points per spline segment.
d (float): Distance ahead for estimating center of view.
n_clusters (int): Number of GMM clusters.
closed (bool): Whether to close the path.
Returns:
List[Camera]: Smooth path of Camera objects.
"""
# Extract positions and rotations
positions = np.array([cam.T for cam in existing_cameras])
rotations = np.array([R.from_matrix(cam.R).as_quat() for cam in existing_cameras])
# === Normalize positions
mean_pos = np.mean(positions, axis=0)
scale_pos = np.std(positions, axis=0)
scale_pos[scale_pos == 0] = 1.0 # avoid division by zero
positions_normalized = (positions - mean_pos) / scale_pos
# === Features for clustering (only positions, not rotations)
features = positions_normalized
# === 1. GMM clustering
gmm = GaussianMixture(n_components=n_clusters, covariance_type='full', random_state=42)
cluster_labels = gmm.fit_predict(features)
clusters = {}
cluster_centers = []
for cluster_id in range(n_clusters):
cluster_indices = np.where(cluster_labels == cluster_id)[0]
if len(cluster_indices) == 0:
continue
clusters[cluster_id] = cluster_indices
cluster_center = np.mean(features[cluster_indices], axis=0)
cluster_centers.append(cluster_center)
cluster_centers = np.stack(cluster_centers)
# === 2. Remap cluster centers to nearest existing cameras
if False:
mapped_centers = []
for center in cluster_centers:
dists = np.linalg.norm(features - center, axis=1)
nearest_idx = np.argmin(dists)
mapped_centers.append(features[nearest_idx])
mapped_centers = np.stack(mapped_centers)
cluster_centers = mapped_centers
# === 3. Solve TSP between mapped cluster centers
cluster_order = solve_tsp_2opt(cluster_centers)
# === 4. For each cluster, solve TSP inside cluster
final_indices = []
for cluster_id in cluster_order:
cluster_indices = clusters[cluster_id]
cluster_positions = features[cluster_indices]
if len(cluster_positions) == 1:
final_indices.append(cluster_indices[0])
continue
local_order = solve_tsp_nearest_neighbor(cluster_positions)
ordered_cluster_indices = cluster_indices[local_order]
final_indices.extend(ordered_cluster_indices)
ordered_cameras = [existing_cameras[i] for i in final_indices]
ordered_positions = positions_normalized[final_indices]
# === 5. Subsample uniformly
idx = np.linspace(0, len(ordered_positions) - 1, n_selected).astype(int)
sampled_positions = ordered_positions[idx]
sampled_cameras = [ordered_cameras[i] for i in idx]
# === 6. Prepare for Catmull-Rom spline
if closed:
sampled_positions = np.vstack([sampled_positions[-1], sampled_positions, sampled_positions[0], sampled_positions[1]])
else:
sampled_positions = np.vstack([sampled_positions[0], sampled_positions, sampled_positions[-1], sampled_positions[-1]])
# === 7. Smooth path positions
path_positions = []
for i in range(1, len(sampled_positions) - 2):
segment = catmull_rom_spline(sampled_positions[i-1], sampled_positions[i], sampled_positions[i+1], sampled_positions[i+2], n_points_per_segment)
path_positions.append(segment)
path_positions = np.concatenate(path_positions, axis=0)
# === 8. Denormalize
path_positions = path_positions * scale_pos + mean_pos
# === 9. SLERP for rotations
rotations = R.from_matrix([cam.R for cam in sampled_cameras])
key_times = np.linspace(0, 1, len(rotations))
slerp = Slerp(key_times, rotations)
query_times = np.linspace(0, 1, len(path_positions))
interpolated_rotations = slerp(query_times)
# === 10. Generate Camera objects
reference_cam = existing_cameras[0]
smooth_cameras = []
for i, pos in enumerate(path_positions):
R_interp = interpolated_rotations[i].as_matrix()
smooth_cameras.append(Camera(
R=R_interp,
T=pos,
FoVx=reference_cam.FoVx,
FoVy=reference_cam.FoVy,
resolution=(reference_cam.image_width, reference_cam.image_height),
colmap_id=-1,
depth_params=None,
image=Image.fromarray(np.zeros((reference_cam.image_height, reference_cam.image_width, 3), dtype=np.uint8)),
invdepthmap=None,
image_name=f"clustered_smooth_path_i={i}",
uid=i
))
return smooth_cameras
# def generate_clustered_path(existing_cameras: List[Camera],
# n_points_per_segment: int = 20,
# d: float = 2.0,
# n_clusters: int = 5,
# closed: bool = False) -> List[Camera]:
# """
# Generate a smooth camera path using GMM clustering and TSP on cluster centers.
# Args:
# existing_cameras (List[Camera]): List of input cameras.
# n_points_per_segment (int): Number of interpolated points per spline segment.
# d (float): Distance ahead for estimating center of view.
# n_clusters (int): Number of GMM clusters (zones).
# closed (bool): Whether to close the path.
# Returns:
# List[Camera]: Smooth path of Camera objects.
# """
# # Extract positions and rotations
# positions = np.array([cam.T for cam in existing_cameras])
# # === Normalize positions
# mean_pos = np.mean(positions, axis=0)
# scale_pos = np.std(positions, axis=0)
# scale_pos[scale_pos == 0] = 1.0
# positions_normalized = (positions - mean_pos) / scale_pos
# # === 1. GMM clustering (only positions)
# gmm = GaussianMixture(n_components=n_clusters, covariance_type='full', random_state=42)
# cluster_labels = gmm.fit_predict(positions_normalized)
# cluster_centers = []
# for cluster_id in range(n_clusters):
# cluster_indices = np.where(cluster_labels == cluster_id)[0]
# if len(cluster_indices) == 0:
# continue
# cluster_center = np.mean(positions_normalized[cluster_indices], axis=0)
# cluster_centers.append(cluster_center)
# cluster_centers = np.stack(cluster_centers)
# # === 2. Solve TSP between cluster centers
# cluster_order = solve_tsp_2opt(cluster_centers)
# # === 3. Reorder cluster centers
# ordered_centers = cluster_centers[cluster_order]
# # === 4. Prepare Catmull-Rom spline
# if closed:
# ordered_centers = np.vstack([ordered_centers[-1], ordered_centers, ordered_centers[0], ordered_centers[1]])
# else:
# ordered_centers = np.vstack([ordered_centers[0], ordered_centers, ordered_centers[-1], ordered_centers[-1]])
# # === 5. Generate smooth path positions
# path_positions = []
# for i in range(1, len(ordered_centers) - 2):
# segment = catmull_rom_spline(ordered_centers[i-1], ordered_centers[i], ordered_centers[i+1], ordered_centers[i+2], n_points_per_segment)
# path_positions.append(segment)
# path_positions = np.concatenate(path_positions, axis=0)
# # === 6. Denormalize back
# path_positions = path_positions * scale_pos + mean_pos
# # === 7. Generate dummy rotations (constant forward facing)
# reference_cam = existing_cameras[0]
# default_rotation = R.from_matrix(reference_cam.R)
# # For simplicity, fixed rotation for all
# smooth_cameras = []
# for i, pos in enumerate(path_positions):
# R_interp = default_rotation.as_matrix()
# smooth_cameras.append(Camera(
# R=R_interp,
# T=pos,
# FoVx=reference_cam.FoVx,
# FoVy=reference_cam.FoVy,
# resolution=(reference_cam.image_width, reference_cam.image_height),
# colmap_id=-1,
# depth_params=None,
# image=Image.fromarray(np.zeros((reference_cam.image_height, reference_cam.image_width, 3), dtype=np.uint8)),
# invdepthmap=None,
# image_name=f"cluster_path_i={i}",
# uid=i
# ))
# return smooth_cameras
from typing import List
import numpy as np
from sklearn.cluster import KMeans
from scipy.spatial.transform import Rotation as R, Slerp
from PIL import Image
def generate_clustered_path(existing_cameras: List[Camera],
n_points_per_segment: int = 20,
d: float = 2.0,
n_clusters: int = 5,
closed: bool = False) -> List[Camera]:
"""
Generate a smooth camera path using K-Means clustering and TSP on cluster centers.
Args:
existing_cameras (List[Camera]): List of input cameras.
n_points_per_segment (int): Number of interpolated points per spline segment.
d (float): Distance ahead for estimating center of view.
n_clusters (int): Number of KMeans clusters (zones).
closed (bool): Whether to close the path.
Returns:
List[Camera]: Smooth path of Camera objects.
"""
# Extract positions
positions = np.array([cam.T for cam in existing_cameras])
# === Normalize positions
mean_pos = np.mean(positions, axis=0)
scale_pos = np.std(positions, axis=0)
scale_pos[scale_pos == 0] = 1.0
positions_normalized = (positions - mean_pos) / scale_pos
# === 1. K-Means clustering (only positions)
kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init='auto')
cluster_labels = kmeans.fit_predict(positions_normalized)
cluster_centers = []
for cluster_id in range(n_clusters):
cluster_indices = np.where(cluster_labels == cluster_id)[0]
if len(cluster_indices) == 0:
continue
cluster_center = np.mean(positions_normalized[cluster_indices], axis=0)
cluster_centers.append(cluster_center)
cluster_centers = np.stack(cluster_centers)
# === 2. Solve TSP between cluster centers
cluster_order = solve_tsp_2opt(cluster_centers)
# === 3. Reorder cluster centers
ordered_centers = cluster_centers[cluster_order]
# === 4. Prepare Catmull-Rom spline
if closed:
ordered_centers = np.vstack([ordered_centers[-1], ordered_centers, ordered_centers[0], ordered_centers[1]])
else:
ordered_centers = np.vstack([ordered_centers[0], ordered_centers, ordered_centers[-1], ordered_centers[-1]])
# === 5. Generate smooth path positions
path_positions = []
for i in range(1, len(ordered_centers) - 2):
segment = catmull_rom_spline(ordered_centers[i-1], ordered_centers[i], ordered_centers[i+1], ordered_centers[i+2], n_points_per_segment)
path_positions.append(segment)
path_positions = np.concatenate(path_positions, axis=0)
# === 6. Denormalize back
path_positions = path_positions * scale_pos + mean_pos
# === 7. Generate dummy rotations (constant forward facing)
reference_cam = existing_cameras[0]
default_rotation = R.from_matrix(reference_cam.R)
# For simplicity, fixed rotation for all
smooth_cameras = []
for i, pos in enumerate(path_positions):
R_interp = default_rotation.as_matrix()
smooth_cameras.append(Camera(
R=R_interp,
T=pos,
FoVx=reference_cam.FoVx,
FoVy=reference_cam.FoVy,
resolution=(reference_cam.image_width, reference_cam.image_height),
colmap_id=-1,
depth_params=None,
image=Image.fromarray(np.zeros((reference_cam.image_height, reference_cam.image_width, 3), dtype=np.uint8)),
invdepthmap=None,
image_name=f"cluster_path_i={i}",
uid=i
))
return smooth_cameras
def visualize_image_with_points(image, points):
"""
Visualize an image with points overlaid on top. This is useful for correspondences visualizations
Parameters:
- image: PIL Image object
- points: Numpy array of shape [N, 2] containing (x, y) coordinates of points
Returns:
- None (displays the visualization)
"""
# Convert PIL image to numpy array
img_array = np.array(image)
# Create a figure and axis
fig, ax = plt.subplots(figsize=(7,7))
# Display the image
ax.imshow(img_array)
# Scatter plot the points on top of the image
ax.scatter(points[:, 0], points[:, 1], color='red', marker='o', s=1)
# Show the plot
plt.show()
def visualize_correspondences(image1, points1, image2, points2):
"""
Visualize two images concatenated horizontally with key points and correspondences.
Parameters:
- image1: PIL Image object (left image)
- points1: Numpy array of shape [N, 2] containing (x, y) coordinates of key points for image1
- image2: PIL Image object (right image)
- points2: Numpy array of shape [N, 2] containing (x, y) coordinates of key points for image2
Returns:
- None (displays the visualization)
"""
# Concatenate images horizontally
concatenated_image = np.concatenate((np.array(image1), np.array(image2)), axis=1)
# Create a figure and axis
fig, ax = plt.subplots(figsize=(10,10))
# Display the concatenated image
ax.imshow(concatenated_image)
# Plot key points on the left image
ax.scatter(points1[:, 0], points1[:, 1], color='red', marker='o', s=10)
# Plot key points on the right image
ax.scatter(points2[:, 0] + image1.width, points2[:, 1], color='blue', marker='o', s=10)
# Draw lines connecting corresponding key points
for i in range(len(points1)):
ax.plot([points1[i, 0], points2[i, 0] + image1.width], [points1[i, 1], points2[i, 1]])#, color='green')
# Show the plot
plt.show()