[3d object detection] CARLA에서 데이터 수집

[이승현]

사용자 환경

os: window 11

CPU: i9-149000KF
RAM: 32G
GPU: RTX4070SUPERTi

목표: carla에서 포인트클라우드 저장하는법과 동시에 가상환경에서 존재하는 모든 객체의 3d바운딩 박스정보를 저장하는 방법을 구현 

•  
•  
•  

칼라 설치 방법 

https://carla-ue5.readthedocs.io/en/latest/start_quickstart/

데이터를 추출한 모습

pointcloud를 추출한 모습

추출한 포인트 클라우드 시각화

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182

class CameraManager(object):     def __init__(self, parent_actor, hud, gamma_correction):         self.sensor = None         self.surface = None         self._parent = parent_actor         self.hud = hud         self.recording = False         bound_x = 0.5 + self._parent.bounding_box.extent.x         bound_y = 0.5 + self._parent.bounding_box.extent.y         bound_z = 0.5 + self._parent.bounding_box.extent.z         Attachment = carla.AttachmentType           if not self._parent.type_id.startswith("walker.pedestrian"):             self._camera_transforms = [                 (carla.Transform(carla.Location(x=-2.0*bound_x, y=+0.0*bound_y, z=2.0*bound_z), carla.Rotation(pitch=8.0)), Attachment.SpringArmGhost),                 (carla.Transform(carla.Location(x=+0.8*bound_x, y=+0.0*bound_y, z=1.3*bound_z)), Attachment.Rigid),                 (carla.Transform(carla.Location(x=+1.9*bound_x, y=+1.0*bound_y, z=1.2*bound_z)), Attachment.SpringArmGhost),                 (carla.Transform(carla.Location(x=-2.8*bound_x, y=+0.0*bound_y, z=4.6*bound_z), carla.Rotation(pitch=6.0)), Attachment.SpringArmGhost),                 (carla.Transform(carla.Location(x=-1.0, y=-1.0*bound_y, z=0.4*bound_z)), Attachment.Rigid)]         else:             self._camera_transforms = [                 (carla.Transform(carla.Location(x=-2.5, z=0.0), carla.Rotation(pitch=-8.0)), Attachment.SpringArmGhost),                 (carla.Transform(carla.Location(x=1.6, z=1.7)), Attachment.Rigid),                 (carla.Transform(carla.Location(x=2.5, y=0.5, z=0.0), carla.Rotation(pitch=-8.0)), Attachment.SpringArmGhost),                 (carla.Transform(carla.Location(x=-4.0, z=2.0), carla.Rotation(pitch=6.0)), Attachment.SpringArmGhost),                 (carla.Transform(carla.Location(x=0, y=-2.5, z=-0.0), carla.Rotation(yaw=90.0)), Attachment.Rigid)]           self.transform_index = 1         self.sensors = [             ['sensor.camera.rgb', cc.Raw, 'Camera RGB', {}],             ['sensor.camera.depth', cc.Raw, 'Camera Depth (Raw)', {}],             ['sensor.camera.depth', cc.Depth, 'Camera Depth (Gray Scale)', {}],             ['sensor.camera.depth', cc.LogarithmicDepth, 'Camera Depth (Logarithmic Gray Scale)', {}],             ['sensor.camera.semantic_segmentation', cc.Raw, 'Camera Semantic Segmentation (Raw)', {}],             ['sensor.camera.semantic_segmentation', cc.CityScapesPalette, 'Camera Semantic Segmentation (CityScapes Palette)', {}],             ['sensor.camera.instance_segmentation', cc.Raw, 'Camera Instance Segmentation (Raw)', {}],             ['sensor.lidar.ray_cast', None, 'Lidar (Ray-Cast)', {'range': '50'}],             ['sensor.lidar.ray_cast_semantic', None, 'Semantic Lidar (Ray-Cast)', {'range': '50'}],             ['sensor.camera.rgb', cc.Raw, 'Camera RGB Distorted',                 {'lens_circle_multiplier': '3.0',                 'lens_circle_falloff': '3.0',                 'chromatic_aberration_intensity': '0.5',                 'chromatic_aberration_offset': '0'}],             ['sensor.camera.optical_flow', cc.Raw, 'Optical Flow', {}],             ['sensor.camera.normals', cc.Raw, 'Camera Normals', {}],         ]         world = self._parent.get_world()         bp_library = world.get_blueprint_library()         for item in self.sensors:             bp = bp_library.find(item[0])             if item[0].startswith('sensor.camera'):                 bp.set_attribute('image_size_x', str(hud.dim[0]))                 bp.set_attribute('image_size_y', str(hud.dim[1]))                 if bp.has_attribute('gamma'):                     bp.set_attribute('gamma', str(gamma_correction))                 for attr_name, attr_value in item[3].items():                     bp.set_attribute(attr_name, attr_value)             elif item[0].startswith('sensor.lidar'):                 self.lidar_range = 50                   for attr_name, attr_value in item[3].items():                     bp.set_attribute(attr_name, attr_value)                     if attr_name == 'range':                         self.lidar_range = float(attr_value)               item.append(bp)         self.index = None       def toggle_camera(self):         self.transform_index = (self.transform_index + 1) % len(self._camera_transforms)         self.set_sensor(self.index, notify=False, force_respawn=True)       def set_sensor(self, index, notify=True, force_respawn=False):         index = index % len(self.sensors)         needs_respawn = True if self.index is None else \             (force_respawn or (self.sensors[index][2] != self.sensors[self.index][2]))         if needs_respawn:             if self.sensor is not None:                 self.sensor.destroy()                 self.surface = None             self.sensor = self._parent.get_world().spawn_actor(                 self.sensors[index][-1],                 self._camera_transforms[self.transform_index][0],                 attach_to=self._parent,                 attachment_type=self._camera_transforms[self.transform_index][1])             # We need to pass the lambda a weak reference to self to avoid             # circular reference.             weak_self = weakref.ref(self)             self.sensor.listen(lambda image: CameraManager._parse_image(weak_self, image))         if notify:             self.hud.notification(self.sensors[index][2])         self.index = index       def next_sensor(self):         self.set_sensor(self.index + 1)       def toggle_recording(self):         self.recording = not self.recording         self.hud.notification('Recording %s' % ('On' if self.recording else 'Off'))       def render(self, display):         if self.surface is not None:             display.blit(self.surface, (0, 0))       @staticmethod     def _parse_image(weak_self, image):         self = weak_self()         if not self:             return         if self.sensors[self.index][0] == 'sensor.lidar.ray_cast':             # === [ADD] 동기화 저장: LIDAR ===             if self.recording:                 ts = round(image.timestamp, 3)                 with buffer_lock:                     if ts not in sync_buffer:                         sync_buffer[ts] = {}                     sync_buffer[ts]["lidar"] = image                     # RGB가 이미 오면, 같이 저장                     if "rgb" in sync_buffer[ts]:                         sync_buffer[ts]["rgb"].save_to_disk(os.path.join(SYNC_OUT_DIR, "rgb", f"{ts:.3f}.png"))                         image.save_to_disk(os.path.join(SYNC_OUT_DIR, "lidar", f"{ts:.3f}.ply"))                         save_bounding_boxes(self._parent.get_world(), ts)                         del sync_buffer[ts]             # =========================================             points = np.frombuffer(image.raw_data, dtype=np.dtype('f4'))             points = np.reshape(points, (int(points.shape[0] / 4), 4))             lidar_data = np.array(points[:, :2])             lidar_data *= min(self.hud.dim) / (2.0 * self.lidar_range)             lidar_data += (0.5 * self.hud.dim[0], 0.5 * self.hud.dim[1])             lidar_data = np.fabs(lidar_data)  # pylint: disable=E1111             lidar_data = lidar_data.astype(np.int32)             lidar_data = np.reshape(lidar_data, (-1, 2))             lidar_img_size = (self.hud.dim[0], self.hud.dim[1], 3)             lidar_img = np.zeros((lidar_img_size), dtype=np.uint8)             lidar_img[tuple(lidar_data.T)] = (255, 255, 255)             self.surface = pygame.surfarray.make_surface(lidar_img)         elif self.sensors[self.index][0] == 'sensor.lidar.ray_cast_semantic':             points = np.frombuffer(image.raw_data, dtype=np.dtype('f4'))             points = np.reshape(points, (int(points.shape[0] / 6), 6))             lidar_data = np.array(points[:, :2])             lidar_data *= min(self.hud.dim) / (2.0 * self.lidar_range)             lidar_data += (0.5 * self.hud.dim[0], 0.5 * self.hud.dim[1])             lidar_data = lidar_data.astype(np.int32)             lidar_data = np.reshape(lidar_data, (-1, 2))             lidar_img_size = (self.hud.dim[0], self.hud.dim[1], 3)             lidar_img = np.zeros((lidar_img_size), dtype=np.uint8)             for i in range(len(image)):                 point = lidar_data[i]                 lidar_tag = image[i].object_tag                 lidar_img[tuple(point.T)] = OBJECT_TO_COLOR[int(lidar_tag)]             self.surface = pygame.surfarray.make_surface(lidar_img)         elif self.sensors[self.index][0].startswith('sensor.camera.optical_flow'):             image = image.get_color_coded_flow()             array = np.frombuffer(image.raw_data, dtype=np.dtype("uint8"))             array = np.reshape(array, (image.height, image.width, 4))             array = array[:, :, :3]             array = array[:, :, ::-1]             self.surface = pygame.surfarray.make_surface(array.swapaxes(0, 1))         else:             image.convert(self.sensors[self.index][1])             array = np.frombuffer(image.raw_data, dtype=np.dtype("uint8"))             array = np.reshape(array, (image.height, image.width, 4))             array = array[:, :, :3]             array = array[:, :, ::-1]             self.surface = pygame.surfarray.make_surface(array.swapaxes(0, 1))         if self.recording:             image.save_to_disk('_out/%08d' % image.frame)         # === [ADD] 동기화 저장: RGB === 내가 추가한 부분분         if self.recording:             # 타임스탬프를 소수점 3자리까지, 동기화 buffer에 추가             ts = round(image.timestamp, 3)             with buffer_lock:                 if ts not in sync_buffer:                     sync_buffer[ts] = {}                 sync_buffer[ts]["rgb"] = image                 # 라이다 데이터가 이미 오면, 같이 저장                 if "lidar" in sync_buffer[ts]:                     image.save_to_disk(os.path.join(SYNC_OUT_DIR, "rgb", f"{ts:.3f}.png"))                     sync_buffer[ts]["lidar"].save_to_disk(os.path.join(SYNC_OUT_DIR, "lidar", f"{ts:.3f}.ply"))                     save_bounding_boxes(self._parent.get_world(), ts)                     del sync_buffer[ts]         # ========================================= Colored by Color Scripter

cs

칼라 라이다 상세정보

https://carla.readthedocs.io/en/latest/ref_sensors/#lidar-sensor

3. 정리표 (현재 manual_control.py의 라이다 사양)

파라미터	값	비고
센서 이름	sensor.lidar.ray_cast	CARLA 내장 Lidar 센서
range	50(m)	코드에서 명시
channels	32	default 값
points_per_second	56000	default 값
rotation_frequency	10(Hz)	default 값
upper_fov	+10	default 값
lower_fov	-30	default 값

문제점: LiDAR와 Camera 이미지가 같은 프레임(동일 타임스탬프)에 저장되지 않음 학습 데이터셋 제작시 반드시 동기화가 필요함( 타음스테프 형식으로 저장을 시도)

2) 3D 바운딩박스/정답 라벨

• CARLA 내부적으로는 모든 객체(차량, 보행자 등)의 위치/크기 정보를 가지고 있음 이 정보를 추출하여 저장하는 코드를 현재 진행중

3) 센서 설치 위치와 각도 문제 LiDAR, Camera의 수, 위치, 각도 를 정해야 하는데 현재는 한개만 설치되어있음

이번주에 해결하여 훈련데이터셋을 만들어 볼 예정 

[Sungryul Lee]

1. carla의 lidar모델의 파라미터를 KITTI, waymo lidar의 특성파라미터와 동일하게 설정하고 데이터 저장할것
2. 3d bbox 정보 저장
3. 3d bbox 정보를 포인트 클라우드에 시각화