Skip to content

Commit

Permalink
fix another bug: visibility was checked based on wrong vertex due to …
Browse files Browse the repository at this point in the history 
…reordering
  • Loading branch information
@tlpss
tlpss committed Dec 5, 2023
1 parent 9407c1a commit 61bfad6
Show file tree
Hide file tree
Showing 2 changed files with 110 additions and 55 deletions.
94 changes: 65 additions & 29 deletions synthetic-cloth-data/synthetic_cloth_data/synthetic_images/scene_builder/annotator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,38 @@
CATEGORY_NAME_TO_KEYPOINTS_DICT,
CLOTH_TYPE_TO_COCO_CATEGORY_ID,
SHORTS_KEYPOINTS,
TOWEL_KEYPOINTS,
TSHIRT_KEYPOINTS,
)


def get_3D_and_2D_keypoints_from_vertices(
cloth_object: bpy.types.Object,
keypoint_vertex_dict: dict,
category_keypoints: List[str],
image_width: int,
image_height: int,
):
# get the 3D coordinates of the keypoints, in the desired order for the coco category
keypoints_3D = [
cloth_object.matrix_world @ cloth_object.data.vertices[keypoint_vertex_dict[keypoint_name]].co
for keypoint_name in category_keypoints
]
scene = bpy.context.scene
camera = bpy.context.scene.camera

keypoints_2D = np.array([np.array(world_to_camera_view(scene, camera, corner))[:2] for corner in keypoints_3D])

# flip y-axis because blender uses y-up and we use y-down (as does coco)
keypoints_2D[:, 1] = 1.0 - keypoints_2D[:, 1]

# scale keypoints to pixel coordinates
keypoints_2D[:, 0] = keypoints_2D[:, 0] * image_width
keypoints_2D[:, 1] = keypoints_2D[:, 1] * image_height

return keypoints_2D, keypoints_3D


@dataclasses.dataclass
class COCOAnnotatorConfig:
# N-ring of vertices to check for visibility of keypoints.
Expand Down Expand Up @@ -66,31 +94,26 @@ def create_coco_annotations( # noqa: C901
# so no annotation needs to be added.
return

# get the 3D coordinates of the keypoints, in the desired order for the coco category
category_keypoints = CATEGORY_NAME_TO_KEYPOINTS_DICT[cloth_type.name]
keypoints_3D = [
cloth_object.matrix_world @ cloth_object.data.vertices[keypoint_vertex_dict[keypoint_name]].co
for keypoint_name in category_keypoints
]
scene = bpy.context.scene
camera = bpy.context.scene.camera

keypoints_2D = np.array([np.array(world_to_camera_view(scene, camera, corner))[:2] for corner in keypoints_3D])

# flip y-axis because blender uses y-up and we use y-down (as does coco)
keypoints_2D[:, 1] = 1.0 - keypoints_2D[:, 1]

# scale keypoints to pixel coordinates
keypoints_2D[:, 0] = keypoints_2D[:, 0] * image_width
keypoints_2D[:, 1] = keypoints_2D[:, 1] * image_height
keypoints_2D, keypoints_3D = get_3D_and_2D_keypoints_from_vertices(
cloth_object, keypoint_vertex_dict, category_keypoints, image_width, image_height
)

# order keypoints to deal with symmetries.
# uses the 2D pixel locations, so that it can also be applied on the 2D keypoints from the real data.
# but we need to make sure that the 2D keypoints are in the same order as the 3D keypoints and that the keypoint_vertex_dict is also updated.
if cloth_type == "TOWEL":
keypoints_2D, keypoints_3D = _order_towel_keypoints(keypoints_2D, keypoints_3D, bbox)
keypoints_2D, keypoints_3D, keypoint_vertex_dict = _order_towel_keypoints(
keypoints_2D, keypoints_3D, keypoint_vertex_dict, bbox
)
if cloth_type == "TSHIRT":
keypoints_2D, keypoints_3D = _order_tshirt_keypoints(keypoints_2D, keypoints_3D, bbox)
keypoints_2D, keypoints_3D, keypoint_vertex_dict = _order_tshirt_keypoints(
keypoints_2D, keypoints_3D, keypoint_vertex_dict, bbox
)
if cloth_type == "SHORTS":
keypoints_2D = order_shorts_keypoints(keypoints_2D, keypoints_3D, bbox)
keypoints_2D, keypoints_3D, keypoint_vertex_dict = order_shorts_keypoints(
keypoints_2D, keypoints_3D, keypoint_vertex_dict, bbox
)

# check if cloth object has a solidify modifier and remove it temporarily because it affects the ray cast and hence the visibility check.
solidify_modifier = None
Expand Down Expand Up @@ -131,6 +154,7 @@ def create_coco_annotations( # noqa: C901

# we set keypoints outside the image to be "not labeled"
if px < 0 or py < 0 or px > image_width or py > image_height:
print("keypoint outside image")
visible_flag = 0
px = 0.0
py = 0.0
Expand All @@ -142,9 +166,9 @@ def create_coco_annotations( # noqa: C901

# for debugging:
# add 3D sphere around each keypoint
# if visible_flag == 2:
# bpy.ops.mesh.primitive_uv_sphere_add(radius=0.01, location=keypoint_3D)
# bpy.context.object.name = f"keypoint_{TSHIRT_KEYPOINTS[keypoint_idx]}"
if visible_flag == 2:
bpy.ops.mesh.primitive_uv_sphere_add(radius=0.01, location=keypoint_3D)
bpy.context.object.name = f"keypoint_{category_keypoints[keypoint_idx]}"

# add the solidifier back if required
if solidify_modifier is not None:
Expand All @@ -170,7 +194,7 @@ def create_coco_annotations( # noqa: C901
json.dump(annotation.dict(exclude_none=True), file, indent=4)


def _order_towel_keypoints(keypoints_2D, keypoints_3D, bbox):
def _order_towel_keypoints(keypoints_2D, keypoints_3D, vertex_dict, bbox):
x_min, y_min, width, height = bbox

# keypoints are in cyclical order but we need to break symmetries by having a starting point in the image viewpoint
Expand Down Expand Up @@ -202,13 +226,16 @@ def _order_towel_keypoints(keypoints_2D, keypoints_3D, bbox):

new_keypoints_2D = [keypoints_2D[i] for i in order]
new_keypoints_3D = [keypoints_3D[i] for i in order]
vertices = list(vertex_dict.values())
new_vertices = [vertices[i] for i in order]
new_vertex_dict = dict(zip(TOWEL_KEYPOINTS, new_vertices))

keypoints_2D = new_keypoints_2D
keypoints_3D = new_keypoints_3D
return keypoints_2D, keypoints_3D
return keypoints_2D, keypoints_3D, new_vertex_dict


def _order_tshirt_keypoints(keypoints_2D: np.ndarray, keypoints_3D: List, bbox: tuple):
def _order_tshirt_keypoints(keypoints_2D: np.ndarray, keypoints_3D: List, keypoints_vertex_dict, bbox: tuple):

# left == side of which the waist kp is closest to the bottom left corner of the bbox in 2D.
# simply serves to break symmetries and find adjacent keypoints, does not correspond with human notion of left and right,
Expand Down Expand Up @@ -241,11 +268,16 @@ def _order_tshirt_keypoints(keypoints_2D: np.ndarray, keypoints_3D: List, bbox:
# https://stackoverflow.com/questions/21288044/row-exchange-in-numpy
keypoints_2D[[idx, right_idx]] = keypoints_2D[[right_idx, idx]]
keypoints_3D[idx], keypoints_3D[right_idx] = keypoints_3D[right_idx], keypoints_3D[idx]

return keypoints_2D, keypoints_3D
keypoints_vertex_dict[keypoint], keypoints_vertex_dict[TSHIRT_KEYPOINTS[right_idx]] = (
keypoints_vertex_dict[TSHIRT_KEYPOINTS[right_idx]],
keypoints_vertex_dict[keypoint],
)
return keypoints_2D, keypoints_3D, keypoints_vertex_dict


def order_shorts_keypoints(keypoints_2D: np.ndarray, keypoints_3D: List, bbox: Tuple[int]) -> np.ndarray:
def order_shorts_keypoints(
keypoints_2D: np.ndarray, keypoints_3D: List, keypoint_vertices_dict: dict, bbox: Tuple[int]
) -> np.ndarray:
x_min, y_min, width, height = bbox

top_left_bbox_corner = (x_min, y_min)
Expand All @@ -271,4 +303,8 @@ def order_shorts_keypoints(keypoints_2D: np.ndarray, keypoints_3D: List, bbox: T
# https://stackoverflow.com/questions/21288044/row-exchange-in-numpy
keypoints_2D[[idx, right_idx]] = keypoints_2D[[right_idx, idx]]
keypoints_3D[idx], keypoints_3D[right_idx] = keypoints_3D[right_idx], keypoints_3D[idx]
return keypoints_2D
keypoint_vertices_dict[keypoint], keypoint_vertices_dict[SHORTS_KEYPOINTS[right_idx]] = (
keypoint_vertices_dict[SHORTS_KEYPOINTS[right_idx]],
keypoint_vertices_dict[keypoint],
)
return keypoints_2D, keypoints_3D, keypoint_vertices_dict
71 changes: 45 additions & 26 deletions ...-cloth-data/synthetic_cloth_data/synthetic_images/scene_builder/utils/visible_vertices.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def is_vertex_occluded_for_scene_camera(co: Vector, helper_cube_scale: float = 0
if DEBUG:
print(f"hit location: {location}")
bpy.ops.mesh.primitive_ico_sphere_add(
location=location, scale=(helper_cube_scale, helper_cube_scale, helper_cube_scale)
location=co, scale=(helper_cube_scale, helper_cube_scale, helper_cube_scale)
)

# remove the auxiliary cube
Expand Down Expand Up @@ -82,31 +82,50 @@ def is_point_in_camera_frustum(point: Vector, camera: bpy.types.Object) -> bool:

if __name__ == "__main__":
"""quick test for the object visibility code."""
import airo_blender as ab
import numpy as np

# delete default cube
bpy.ops.object.delete()

camera = bpy.data.objects["Camera"]
camera.location = (-10, 0, 0)
camera.rotation_euler = (np.pi / 2, 0, -np.pi / 2)

# add 100 random spheres to scene
objects = []
for i in range(100):
scale = 0.1

z, y = np.random.random(2)
z, y = z * 10 - 5, y * 10 - 5
bpy.ops.mesh.primitive_cube_add(location=(10, y, z), scale=(scale, scale, scale))
cube_obj = bpy.context.active_object
objects.append(cube_obj)

bpy.ops.mesh.primitive_cube_add(location=(10, 0, 1.1), scale=(0.1, 0.1, 0.1))
cube_obj = bpy.context.active_object
objects.append(cube_obj)

for obj in objects:
occluded = is_object_occluded_for_scene_camera(obj)
if occluded:
ab.add_material(obj, (1, 0, 0))
else:
ab.add_material(obj, (0, 1, 0))
camera.location = (-2, 0, 2)
camera.rotation_euler = (np.pi / 4, 0, -np.pi / 2)

# # add 100 random spheres to scene
# objects = []
# for i in range(100):
# scale = 0.1

# z, y = np.random.random(2)
# z, y = z * 10 - 5, y * 10 - 5
# bpy.ops.mesh.primitive_cube_add(location=(10, y, z), scale=(scale, scale, scale))
# cube_obj = bpy.context.active_object
# objects.append(cube_obj)

# bpy.ops.mesh.primitive_cube_add(location=(10, 0, 1.1), scale=(0.1, 0.1, 0.1))
# cube_obj = bpy.context.active_object
# objects.append(cube_obj)

# for obj in objects:
# occluded = is_object_occluded_for_scene_camera(obj)
# if occluded:
# ab.add_material(obj, (1, 0, 0))
# else:
# ab.add_material(obj, (0, 1, 0))

# load cloth mesh
mesh_file = "/fast_storage_2/symlinked_homes/tlips/Documents/Documents/synthetic-cloth-data/synthetic-cloth-data/data/deformed_meshes/TSHIRT/04-single-layer/002488.obj"
bpy.ops.import_scene.obj(filepath=mesh_file, split_mode="OFF") # keep vertex order with split_mode="OFF"
cloth_object = bpy.context.selected_objects[0]
bpy.ops.object.select_all(action="DESELECT")
bpy.context.view_layer.objects.active = cloth_object
cloth_object.select_set(True)

import tqdm

# check if vertices are occluded
for vertex in tqdm.tqdm(cloth_object.data.vertices):
coords = cloth_object.matrix_world @ vertex.co
if not is_vertex_occluded_for_scene_camera(coords):
# add sphere at the vertex
bpy.ops.mesh.primitive_ico_sphere_add(location=coords, scale=(0.0005, 0.0005, 0.0005))

0 comments on commit 61bfad6

Please sign in to comment.