[3d object detection] open3d visualization

[이승현]

open3d 참고 사이트

https://github.com/isl-org/Open3D-ML

GitHub - isl-org/Open3D-ML: An extension of Open3D to address 3D Machine Learning tasks

https://gaussian37.github.io/autodrive-lidar-open3d/

포인트 클라우드 처리를 위한 open3d 사용법 정리

open3d visualization 문서

https://www.open3d.org/docs/latest/python_api/open3d.visualization.html

open3d.visualization - Open3D primary (unknown) documentation

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117

#point cloud 시각화 코드드   import open3d import torch import numpy as np import pickle import os   # 클래스별 컬러맵 설정 box_colormap = {     1: [0, 1, 0],  # Car (초록)     2: [0, 1, 1],  # Pedestrian (민트색)     3: [1, 1, 0],  # Cyclist (노란색) }   def draw_scenes(points, gt_boxes=None, ref_boxes=None, ref_labels=None, ref_scores=None):     if isinstance(points, torch.Tensor):         points = points.cpu().numpy()     if isinstance(gt_boxes, torch.Tensor):         gt_boxes = gt_boxes.cpu().numpy()     if isinstance(ref_boxes, torch.Tensor):         ref_boxes = ref_boxes.cpu().numpy()       vis = open3d.visualization.Visualizer()     vis.create_window()       vis.get_render_option().point_size = 1.0     vis.get_render_option().background_color = np.zeros(3)       axis_pcd = open3d.geometry.TriangleMesh.create_coordinate_frame(size=1.0, origin=[0, 0, 0])     vis.add_geometry(axis_pcd)       pts = open3d.geometry.PointCloud()     pts.points = open3d.utility.Vector3dVector(points[:, :3])     pts.colors = open3d.utility.Vector3dVector(np.ones((points.shape[0], 3)))     vis.add_geometry(pts)       if gt_boxes is not None and len(gt_boxes) > 0:         vis = draw_box(vis, gt_boxes, color=(0, 0, 1))  #정답(ground truth) 박스를 파란색으로 표시       if ref_boxes is not None and len(ref_boxes) > 0:         vis = draw_box(vis, ref_boxes, ref_labels=ref_labels)  #예측 박스는 클래스별 색상 적용       vis.run()     vis.destroy_window()   def draw_box(vis, boxes, color=(0, 1, 0), ref_labels=None):     for i in range(boxes.shape[0]):         line_set, box3d = translate_boxes_to_open3d_instance(boxes[i])           if ref_labels is not None:             color = box_colormap.get(ref_labels[i], (1, 1, 1))           line_set.paint_uniform_color(color)         vis.add_geometry(line_set)     return vis   def translate_boxes_to_open3d_instance(gt_box):     center = gt_box[0:3]     lwh = gt_box[3:6]     axis_angles = np.array([0, 0, gt_box[6] + 1e-10])     rot = open3d.geometry.get_rotation_matrix_from_axis_angle(axis_angles)# 회전정보보     box3d = open3d.geometry.OrientedBoundingBox(center, rot, lwh)# 3d 박스정보     line_set = open3d.geometry.LineSet.create_from_oriented_bounding_box(box3d)     return line_set, box3d     # 포인트 클라우드에 바운딩 박스(정답,예측)을 그려주는 함수 def process_single_frame(dataset_path, frame_id):      points_path = os.path.join(dataset_path, "velodyne", f"{frame_id}.bin")# 포인트 클라우드     info_path = os.path.join(dataset_path, "kitti_infos_train.pkl") # 정답      result_path = os.path.join(dataset_path, "result.pkl")# 예측       if not os.path.exists(points_path) or not os.path.exists(info_path):         print(f"[ERROR] Missing file: {points_path} or {info_path}")         return       points = np.fromfile(points_path, dtype=np.float32).reshape(-1, 4)       #print(f"Point Cloud 범위: Min {points.min(axis=0)}, Max {points.max(axis=0)}")       # train_info.pkl에서 GT 박스 가져오기     with open(info_path, 'rb') as f:         infos = pickle.load(f)     print(f"정답답",infos[1])     # 현재 frame에 해당하는 데이터 찾기(infos_train에는 lidar_idx로 표시됨)     gt_boxes_lidar = None     for info in infos:         if info["point_cloud"]["lidar_idx"] == frame_id:             gt_boxes_lidar = info["annos"]["gt_boxes_lidar"]             break       if gt_boxes_lidar is None:         print(f"[ERROR] GT Box not found for frame {frame_id}")         return       #print(f"[INFO] GT Boxes Shape: {gt_boxes_lidar.shape}")     #print(f"[INFO] GT Box Sample: {gt_boxes_lidar[:3]}")        #result.pkl에서 예측된 박스 가져오기     with open(result_path, 'rb') as f:         results = pickle.load(f)     print(f"예측 결과값",results[1]) # 확인용     #프레임별로 예측한 결과 시각화 하는 코드     ref_boxes = results[5]['boxes_lidar']     ref_labels = results[5]['name']     ref_scores = results[5]['score']       label_mapping = {"Car": 1, "Pedestrian": 2, "Cyclist": 3}     ref_labels = [label_mapping[label] for label in ref_labels]       print(f"[INFO] Visualizing frame {frame_id}...")     draw_scenes(points, gt_boxes_lidar, ref_boxes=ref_boxes, ref_labels=ref_labels, ref_scores=ref_scores)   if __name__ == "__main__":     dataset_path = r'C:\Users\1\Desktop\train'# 데이터셋 path     frame_id = "000011"  # 원하는 프레임 ID 입력     process_single_frame(dataset_path, frame_id) Colored by Color Scripter

cs

pointcloud 시각화

open3d GUI를 사용한 시각화 코드

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102

#open3d gui 코드(3D바운딩박스 예측,정답 출력력)   import open3d as o3d import numpy as np import pickle import os import open3d.visualization.gui as gui import open3d.visualization.rendering as rendering   # 클래스별 컬러맵 설정 box_colormap = {     "Car": [0, 1, 0],  # 초록색     "Pedestrian": [0, 1, 1],  # 민트색     "Cyclist": [1, 1, 0],  # 노란색 } GT_COLOR = [0, 0, 1]  # 정답(GT) 박스는 파란색     def create_open3d_bounding_box(box, color):     """     3D 바운딩 박스를 Open3D 객체로 변환하는 함수     """     center = box[:3]     lwh = box[3:6]     axis_angles = np.array([0, 0, box[6]])     rot = o3d.geometry.get_rotation_matrix_from_axis_angle(axis_angles)     box3d = o3d.geometry.OrientedBoundingBox(center, rot, lwh)     box3d.color = color     return box3d     def visualize_3d_scene_with_labels(points, gt_boxes, pred_boxes, pred_labels, pred_scores):     """     Open3D GUI를 사용하여 바운딩 박스 위에 클래스와 점수를 표시하는 3D 시각화     """     app = gui.Application.instance     app.initialize()       vis = o3d.visualization.O3DVisualizer("3D Object Detection", 1024, 768)     vis.show_settings = True  # 설정 창 활성화     vis.scene.set_background([0, 0, 0, 1])       # 포인트 클라우드 추가     pcd = o3d.geometry.PointCloud()     pcd.points = o3d.utility.Vector3dVector(points[:, :3])     vis.add_geometry("PointCloud", pcd)       # GT 바운딩 박스 추가     for gt_box in gt_boxes:         box = create_open3d_bounding_box(gt_box, GT_COLOR)         vis.add_geometry(f"GT_{np.random.randint(0, 10000)}", box)       # 예측 바운딩 박스 및 라벨 추가     for i, pred_box in enumerate(pred_boxes):         color = box_colormap.get(pred_labels[i], [1, 1, 1])         box = create_open3d_bounding_box(pred_box, color)         vis.add_geometry(f"Pred_{i}", box)           # 바운딩 박스 위에 클래스 및 점수 추가         text_label = f"{pred_labels[i]} ({pred_scores[i]:.2f})"         label_pos = pred_box[:3] + np.array([0, 0, pred_box[5] / 2 + 0.5])  # 바운딩 박스 위쪽         vis.add_3d_label(label_pos, text_label)           print(f"[INFO] {pred_labels[i]} ({pred_scores[i]:.2f}) at {pred_box[:3]}")     vis.reset_camera_to_default()       app.add_window(vis)     app.run()     def process_single_frame(dataset_path, frame_id):     """     LiDAR 포인트 클라우드와 예측된 바운딩 박스를 로드하여 시각화     """     points_path = os.path.join(dataset_path, "velodyne", f"{frame_id}.bin")     result_path = os.path.join(dataset_path, "result.pkl")       if not os.path.exists(points_path) or not os.path.exists(result_path):         print(f"[ERROR] Missing file: {points_path} or {result_path}")         return       points = np.fromfile(points_path, dtype=np.float32).reshape(-1, 4)       #result.pkl에서 예측된 바운딩 박스 로드     with open(result_path, 'rb') as f:         results = pickle.load(f)       pred_boxes = results[5]['boxes_lidar']     pred_labels = results[5]['name']     pred_scores = results[5]['score']       label_mapping = {"Car": "Car", "Pedestrian": "Pedestrian", "Cyclist": "Cyclist"}     pred_labels = [label_mapping[label] for label in pred_labels]       print(f"[INFO] Visualizing frame {frame_id}...")     visualize_3d_scene_with_labels(points, pred_boxes, pred_boxes, pred_labels, pred_scores)     if __name__ == "__main__":     dataset_path = r'C:\Users\1\Desktop\train'     frame_id = "000011"     process_single_frame(dataset_path, frame_id) Colored by Color Scripter

cs

open3d GUI

o3d.visualization.Visualizer()---> o3d.visualization.O3DVisualizer 변경(Open3D GUI에서 확대/축소, 회전, 시점 변경 기능 추가됨)
create_box_from_dim_with_arrow()-->OrientedBoundingBox 변경(Yaw(회전)이 반영된 정확한 3D 바운딩 박스 생성 기능 추가)
add_3d_label()---->(객체의 클래스 및 신뢰도 Score를 3D 공간에서 바로 확인 가능)

예측 결과 시각화

my_model_0

논문model_0

my_model_1

논문model1

my_model_2

논문model_2

my_model_3

논문model_3

my_model_4

논문model_4

예측 결과 시각화에 사용한 코드

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118

#point cloud 시각화 코드드   import open3d import torch import numpy as np import pickle import os   # 클래스별 컬러맵 설정 box_colormap = {     1: [0, 1, 0],  # Car (초록)     2: [0, 1, 1],  # Pedestrian (민트색)     3: [1, 1, 0],  # Cyclist (노란색) } frame_id = "000004"  # 원하는 프레임 ID 입력 def draw_scenes(points, gt_boxes=None, ref_boxes=None, ref_labels=None, ref_scores=None):     if isinstance(points, torch.Tensor):         points = points.cpu().numpy()     if isinstance(gt_boxes, torch.Tensor):         gt_boxes = gt_boxes.cpu().numpy()     if isinstance(ref_boxes, torch.Tensor):         ref_boxes = ref_boxes.cpu().numpy()       vis = open3d.visualization.Visualizer()     vis.create_window()       vis.get_render_option().point_size = 1.0     vis.get_render_option().background_color = np.zeros(3)       axis_pcd = open3d.geometry.TriangleMesh.create_coordinate_frame(size=1.0, origin=[0, 0, 0])     vis.add_geometry(axis_pcd)       pts = open3d.geometry.PointCloud()     pts.points = open3d.utility.Vector3dVector(points[:, :3])     pts.colors = open3d.utility.Vector3dVector(np.ones((points.shape[0], 3)))     vis.add_geometry(pts)       if gt_boxes is not None and len(gt_boxes) > 0:         vis = draw_box(vis, gt_boxes, color=(0, 0, 1))  #정답(ground truth) 박스를 파란색으로 표시       if ref_boxes is not None and len(ref_boxes) > 0:         vis = draw_box(vis, ref_boxes, ref_labels=ref_labels)  #예측 박스는 클래스별 색상 적용     vis.run()     #vis.capture_screen_image("my_%06d.jpg" % int(frame_id))# 내가 학습한 모델     vis.capture_screen_image("paper_%06d.jpg" % int(frame_id))# 논문 모델델     vis.destroy_window()   def draw_box(vis, boxes, color=(0, 1, 0), ref_labels=None):     for i in range(boxes.shape[0]):         line_set, box3d = translate_boxes_to_open3d_instance(boxes[i])           if ref_labels is not None:             color = box_colormap.get(ref_labels[i], (1, 1, 1))           line_set.paint_uniform_color(color)         vis.add_geometry(line_set)     return vis   def translate_boxes_to_open3d_instance(gt_box):     center = gt_box[0:3]     lwh = gt_box[3:6]     axis_angles = np.array([0, 0, gt_box[6] + 1e-10])     rot = open3d.geometry.get_rotation_matrix_from_axis_angle(axis_angles)# 회전정보보     box3d = open3d.geometry.OrientedBoundingBox(center, rot, lwh)# 3d 박스정보     line_set = open3d.geometry.LineSet.create_from_oriented_bounding_box(box3d)     return line_set, box3d     # 포인트 클라우드에 바운딩 박스(정답,예측)을 그려주는 함수 def process_single_frame(dataset_path, frame_id):      points_path = os.path.join(dataset_path, "velodyne", f"{frame_id}.bin")# 포인트 클라우드     #info_path = os.path.join(dataset_path, "kitti_infos_train.pkl") # 정답 kitti_infos_train.pkl     result_path = os.path.join(dataset_path, "voxelrcnn_test_result.pkl")# 예측 voxelrcnn_test_result.pkl       #if not os.path.exists(points_path) or not os.path.exists(info_path):     #    print(f"[ERROR] Missing file: {points_path} or {info_path}")     #    return       points = np.fromfile(points_path, dtype=np.float32).reshape(-1, 4)       #print(f"Point Cloud 범위: Min {points.min(axis=0)}, Max {points.max(axis=0)}")       # train_info.pkl에서 GT 박스 가져오기     #with open(info_path, 'rb') as f:     #    infos = pickle.load(f)     #print(f"정답답",infos[1])     # 현재 frame에 해당하는 데이터 찾기(infos_train에는 lidar_idx로 표시됨)     gt_boxes_lidar = None     #for info in infos:     #    if info["point_cloud"]["lidar_idx"] == frame_id:     #        gt_boxes_lidar = info["annos"]["gt_boxes_lidar"]     #        break       #if gt_boxes_lidar is None:     #    print(f"[ERROR] GT Box not found for frame {frame_id}")     #    return       #print(f"[INFO] GT Boxes Shape: {gt_boxes_lidar.shape}")     #print(f"[INFO] GT Box Sample: {gt_boxes_lidar[:3]}")        #result.pkl에서 예측된 박스 가져오기     with open(result_path, 'rb') as f:         results = pickle.load(f)     print(f"예측 결과값",results[1]) # 확인용     #프레임별로 예측한 결과 시각화 하는 코드     ref_boxes = results[4]['boxes_lidar']     ref_labels = results[4]['name']     ref_scores = results[4]['score']       label_mapping = {"Car": 1, "Pedestrian": 2, "Cyclist": 3}     ref_labels = [label_mapping[label] for label in ref_labels]       print(f"[INFO] Visualizing frame {frame_id}...")     draw_scenes(points, gt_boxes_lidar, ref_boxes=ref_boxes, ref_labels=ref_labels, ref_scores=ref_scores)   if __name__ == "__main__":     dataset_path = r'C:\Users\1\Desktop\test'# 데이터셋 path          process_single_frame(dataset_path, frame_id) Colored by Color Scripter

cs

[Sungryul Lee]

SOTA 모델 5개이상 찾아서 비교 분석할 것