Add mp4 compression and reading for pinhole and fisheye cameras.

Author: DanielDauner

src/py123d/conversion/dataset_converter_config.py

@@ -41,12 +41,11 @@ class DatasetConverterConfig:
     include_route: bool = False
 
     def __post_init__(self):
-        assert (
-            self.pinhole_camera_store_option != "mp4"
-        ), "MP4 format is not yet supported, but planned for future releases."
+
         assert self.pinhole_camera_store_option in [
             "path",
             "binary",
+            "mp4",
         ], f"Invalid camera store option, got {self.pinhole_camera_store_option}."
 
         assert self.lidar_store_option in [

src/py123d/conversion/datasets/av2/av2_sensor_converter.py

@@ -14,7 +14,7 @@
     find_closest_target_fpath,
     get_slice_with_timestamp_ns,
 )
-from py123d.conversion.log_writer.abstract_log_writer import AbstractLogWriter, LiDARData
+from py123d.conversion.log_writer.abstract_log_writer import AbstractLogWriter, CameraData, LiDARData
 from py123d.conversion.map_writer.abstract_map_writer import AbstractMapWriter
 from py123d.conversion.registry.box_detection_label_registry import AV2SensorBoxDetectionLabel
 from py123d.conversion.registry.lidar_index_registry import AVSensorLiDARIndex
@@ -322,9 +322,9 @@ def _extract_av2_sensor_pinhole_cameras(
     synchronization_df: pd.DataFrame,
     source_log_path: Path,
     dataset_converter_config: DatasetConverterConfig,
-) -> Dict[PinholeCameraType, Tuple[Union[str, bytes], StateSE3]]:
+) -> List[CameraData]:
 
-    camera_dict: Dict[PinholeCameraType, Tuple[Union[str, bytes], StateSE3]] = {}
+    camera_data_list: List[CameraData] = []
     split = source_log_path.parent.name
     log_id = source_log_path.name
 
@@ -351,17 +351,15 @@ def _extract_av2_sensor_pinhole_cameras(
                 absolute_image_path = av2_sensor_data_root / relative_image_path
                 assert absolute_image_path.exists()
 
-                # TODO: Adjust for finer IMU timestamps to correct the camera extrinsic.
-                camera_extrinsic = _row_dict_to_state_se3(row)
-                camera_data = None
-                if dataset_converter_config.pinhole_camera_store_option == "path":
-                    camera_data = str(relative_image_path)
-                elif dataset_converter_config.pinhole_camera_store_option == "binary":
-                    with open(absolute_image_path, "rb") as f:
-                        camera_data = f.read()
-                camera_dict[pinhole_camera_type] = camera_data, camera_extrinsic
-
-    return camera_dict
+                camera_data = CameraData(
+                    camera_type=pinhole_camera_type,
+                    extrinsic=_row_dict_to_state_se3(row),
+                    dataset_root=av2_sensor_data_root,
+                    relative_path=relative_image_path,
+                )
+                camera_data_list.append(camera_data)
+
+    return camera_data_list
 
 
 def _extract_av2_sensor_lidars(

src/py123d/conversion/datasets/kitti360/kitti360_converter.py

@@ -24,7 +24,7 @@
     kittiId2label,
 )
 from py123d.conversion.datasets.kitti360.utils.preprocess_detection import process_detection
-from py123d.conversion.log_writer.abstract_log_writer import AbstractLogWriter, LiDARData
+from py123d.conversion.log_writer.abstract_log_writer import AbstractLogWriter, CameraData, LiDARData
 from py123d.conversion.map_writer.abstract_map_writer import AbstractMapWriter
 from py123d.conversion.registry.box_detection_label_registry import KITTI360BoxDetectionLabel
 from py123d.conversion.registry.lidar_index_registry import Kitti360LiDARIndex
@@ -304,12 +304,9 @@ def convert_log(self, log_index: int, log_writer: AbstractLogWriter) -> None:
                     timestamp=ts_list[valid_idx],
                     ego_state=ego_state_all[idx],
                     box_detections=box_detection_wrapper_all[valid_idx],
-                    traffic_lights=None,
                     pinhole_cameras=pinhole_cameras,
                     fisheye_mei_cameras=fisheye_cameras,
                     lidars=lidars,
-                    scenario_tags=None,
-                    route_lane_group_ids=None,
                 )
 
         log_writer.close()
@@ -724,26 +721,24 @@ def _extract_kitti360_pinhole_cameras(
     camera_calibration: Dict[str, StateSE3],
     kitti360_folders: Dict[str, Path],
     data_converter_config: DatasetConverterConfig,
-) -> Dict[Union[PinholeCameraType, FisheyeMEICameraType], Optional[Tuple[Union[str, bytes], StateSE3]]]:
+) -> List[CameraData]:
 
-    pinhole_camera_dict: Dict[PinholeCameraType, Optional[Tuple[Union[str, bytes], StateSE3]]] = {}
+    pinhole_camera_data_list: List[CameraData] = []
     if data_converter_config.include_pinhole_cameras:
-
         for camera_type, cam_dir_name in KITTI360_PINHOLE_CAMERA_TYPES.items():
             img_path_png = kitti360_folders[DIR_2D_RAW] / log_name / cam_dir_name / "data_rect" / f"{idx:010d}.png"
             camera_extrinsic = camera_calibration[cam_dir_name]
-
             if img_path_png.exists():
-                if data_converter_config.pinhole_camera_store_option == "path":
-                    camera_data = str(img_path_png)
-                elif data_converter_config.pinhole_camera_store_option == "binary":
-                    with open(img_path_png, "rb") as f:
-                        camera_data = f.read()
-            else:
-                camera_data = None
+                pinhole_camera_data_list.append(
+                    CameraData(
+                        camera_type=camera_type,
+                        extrinsic=camera_extrinsic,
+                        dataset_root=kitti360_folders[DIR_ROOT],
+                        relative_path=img_path_png.relative_to(kitti360_folders[DIR_ROOT]),
+                    )
+                )
 
-            pinhole_camera_dict[camera_type] = camera_data, camera_extrinsic
-    return pinhole_camera_dict
+    return pinhole_camera_data_list
 
 
 def _extract_kitti360_fisheye_mei_cameras(
@@ -752,22 +747,23 @@ def _extract_kitti360_fisheye_mei_cameras(
     camera_calibration: Dict[str, StateSE3],
     kitti360_folders: Dict[str, Path],
     data_converter_config: DatasetConverterConfig,
-) -> Dict[Union[PinholeCameraType, FisheyeMEICameraType], Optional[Tuple[Union[str, bytes], StateSE3]]]:
-
-    fisheye_camera_dict: Dict[FisheyeMEICameraType, Optional[Tuple[Union[str, bytes], StateSE3]]] = {}
-    for camera_type, cam_dir_name in KITTI360_FISHEYE_MEI_CAMERA_TYPES.items():
-        img_path_png = kitti360_folders[DIR_2D_RAW] / log_name / cam_dir_name / "data_rgb" / f"{idx:010d}.png"
-        camera_extrinsic = camera_calibration[cam_dir_name]
-        if img_path_png.exists():
-            if data_converter_config.pinhole_camera_store_option == "path":
-                camera_data = str(img_path_png)
-            elif data_converter_config.pinhole_camera_store_option == "binary":
-                with open(img_path_png, "rb") as f:
-                    camera_data = f.read()
-        else:
-            camera_data = None
-        fisheye_camera_dict[camera_type] = camera_data, camera_extrinsic
-    return fisheye_camera_dict
+) -> List[CameraData]:
+
+    fisheye_camera_data_list: List[CameraData] = []
+    if data_converter_config.include_fisheye_mei_cameras:
+        for camera_type, cam_dir_name in KITTI360_FISHEYE_MEI_CAMERA_TYPES.items():
+            img_path_png = kitti360_folders[DIR_2D_RAW] / log_name / cam_dir_name / "data_rgb" / f"{idx:010d}.png"
+            camera_extrinsic = camera_calibration[cam_dir_name]
+            if img_path_png.exists():
+                fisheye_camera_data_list.append(
+                    CameraData(
+                        camera_type=camera_type,
+                        extrinsic=camera_extrinsic,
+                        dataset_root=kitti360_folders[DIR_ROOT],
+                        relative_path=img_path_png.relative_to(kitti360_folders[DIR_ROOT]),
+                    )
+                )
+    return fisheye_camera_data_list
 
 
 def _load_kitti_360_calibration(kitti_360_data_root: Path) -> Dict[str, StateSE3]:

src/py123d/conversion/datasets/nuplan/nuplan_converter.py

@@ -22,7 +22,7 @@
     get_box_detections_for_lidarpc_token_from_db,
     get_nearest_ego_pose_for_timestamp_from_db,
 )
-from py123d.conversion.log_writer.abstract_log_writer import AbstractLogWriter, LiDARData
+from py123d.conversion.log_writer.abstract_log_writer import AbstractLogWriter, CameraData, LiDARData
 from py123d.conversion.map_writer.abstract_map_writer import AbstractMapWriter
 from py123d.conversion.registry.box_detection_label_registry import NuPlanBoxDetectionLabel
 from py123d.conversion.registry.lidar_index_registry import NuPlanLiDARIndex
@@ -356,9 +356,9 @@ def _extract_nuplan_cameras(
     source_log_path: Path,
     nuplan_sensor_root: Path,
     dataset_converter_config: DatasetConverterConfig,
-) -> Dict[PinholeCameraType, Tuple[Union[str, bytes], StateSE3]]:
+) -> List[CameraData]:
 
-    camera_dict: Dict[str, Union[str, bytes]] = {}
+    camera_data_list: List[CameraData] = []
 
     if dataset_converter_config.include_pinhole_cameras:
         log_cam_infos = {camera.token: camera for camera in nuplan_log_db.log.cameras}
@@ -395,18 +395,17 @@ def _extract_nuplan_cameras(
                     c2e = img_e2e @ c2img_e
                     extrinsic = StateSE3.from_transformation_matrix(c2e)
 
-                    # Store camera data, either as path or binary
-                    camera_data: Optional[Union[str, bytes]] = None
-                    if dataset_converter_config.pinhole_camera_store_option == "path":
-                        camera_data = str(filename_jpg)
-                    elif dataset_converter_config.pinhole_camera_store_option == "binary":
-                        with open(filename_jpg, "rb") as f:
-                            camera_data = f.read()
-
                     # Store in dictionary
-                    camera_dict[camera_type] = camera_data, extrinsic
+                    camera_data_list.append(
+                        CameraData(
+                            camera_type=camera_type,
+                            extrinsic=extrinsic,
+                            dataset_root=nuplan_sensor_root,
+                            relative_path=filename_jpg.relative_to(nuplan_sensor_root),
+                        )
+                    )
 
-    return camera_dict
+    return camera_data_list
 
 
 def _extract_nuplan_lidars(

src/py123d/conversion/datasets/nuscenes/nuscenes_converter.py

@@ -1,6 +1,6 @@
 import gc
 from pathlib import Path
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any, Dict, List, Union
 
 import numpy as np
 from pyquaternion import Quaternion
@@ -15,7 +15,7 @@
     NUSCENES_DETECTION_NAME_DICT,
     NUSCENES_DT,
 )
-from py123d.conversion.log_writer.abstract_log_writer import AbstractLogWriter, LiDARData
+from py123d.conversion.log_writer.abstract_log_writer import AbstractLogWriter, CameraData, LiDARData
 from py123d.conversion.map_writer.abstract_map_writer import AbstractMapWriter
 from py123d.conversion.registry.box_detection_label_registry import NuScenesBoxDetectionLabel
 from py123d.conversion.registry.lidar_index_registry import NuScenesLiDARIndex
@@ -388,8 +388,8 @@ def _extract_nuscenes_cameras(
     sample: Dict[str, Any],
     nuscenes_data_root: Path,
     dataset_converter_config: DatasetConverterConfig,
-) -> Dict[PinholeCameraType, Tuple[Union[str, bytes], StateSE3]]:
-    camera_dict: Dict[PinholeCameraType, Tuple[Union[str, bytes], StateSE3]] = {}
+) -> List[CameraData]:
+    camera_data_list: List[CameraData] = []
 
     if dataset_converter_config.include_pinhole_cameras:
         for camera_type, camera_channel in NUSCENES_CAMERA_TYPES.items():
@@ -409,20 +409,20 @@ def _extract_nuscenes_cameras(
             extrinsic_matrix[:3, 3] = translation
             extrinsic = StateSE3.from_transformation_matrix(extrinsic_matrix)
 
-            cam_path = nuscenes_data_root / cam_data["filename"]
+            cam_path = nuscenes_data_root / str(cam_data["filename"])
 
             if cam_path.exists() and cam_path.is_file():
-                if dataset_converter_config.pinhole_camera_store_option == "path":
-                    camera_data = str(cam_path)
-                elif dataset_converter_config.pinhole_camera_store_option == "binary":
-                    with open(cam_path, "rb") as f:
-                        camera_data = f.read()
-                else:
-                    continue
-
-                camera_dict[camera_type] = (camera_data, extrinsic)
+                # camera_dict[camera_type] = (camera_data, extrinsic)
+                camera_data_list.append(
+                    CameraData(
+                        camera_type=camera_type,
+                        extrinsic=extrinsic,
+                        relative_path=cam_path.relative_to(nuscenes_data_root),
+                        dataset_root=nuscenes_data_root,
+                    )
+                )
 
-    return camera_dict
+    return camera_data_list
 
 
 def _extract_nuscenes_lidars(

src/py123d/conversion/datasets/pandaset/pandaset_converter.py

@@ -22,7 +22,7 @@
     read_pkl_gz,
     rotate_pandaset_pose_to_iso_coordinates,
 )
-from py123d.conversion.log_writer.abstract_log_writer import AbstractLogWriter, LiDARData
+from py123d.conversion.log_writer.abstract_log_writer import AbstractLogWriter, CameraData, LiDARData
 from py123d.conversion.map_writer.abstract_map_writer import AbstractMapWriter
 from py123d.conversion.registry.box_detection_label_registry import PandasetBoxDetectionLabel
 from py123d.conversion.registry.lidar_index_registry import PandasetLiDARIndex
@@ -332,9 +332,8 @@ def _extract_pandaset_sensor_camera(
     ego_state_se3: EgoStateSE3,
     camera_poses: Dict[str, List[Dict[str, Dict[str, float]]]],
     dataset_converter_config: DatasetConverterConfig,
-) -> Dict[PinholeCameraType, Tuple[Union[str, bytes], StateSE3]]:
-
-    camera_dict: Dict[PinholeCameraType, Tuple[Union[str, bytes], StateSE3]] = {}
+) -> List[CameraData]:
+    camera_data_list: List[CameraData] = []
     iteration_str = f"{iteration:02d}"
 
     if dataset_converter_config.include_pinhole_cameras:
@@ -346,22 +345,20 @@ def _extract_pandaset_sensor_camera(
 
             camera_pose_dict = camera_poses[camera_name][iteration]
             camera_extrinsic = pandaset_pose_dict_to_state_se3(camera_pose_dict)
-            # camera_extrinsic = rotate_pandaset_pose_to_iso_coordinates(camera_extrinsic)
 
             camera_extrinsic = StateSE3.from_array(
                 convert_absolute_to_relative_se3_array(ego_state_se3.rear_axle_se3, camera_extrinsic.array), copy=True
             )
+            camera_data_list.append(
+                CameraData(
+                    camera_type=camera_type,
+                    extrinsic=camera_extrinsic,
+                    dataset_root=source_log_path.parent,
+                    relative_path=image_abs_path.relative_to(source_log_path.parent),
+                )
+            )
 
-            camera_data = None
-            if dataset_converter_config.pinhole_camera_store_option == "path":
-                pandaset_data_root = source_log_path.parent
-                camera_data = str(image_abs_path.relative_to(pandaset_data_root))
-            elif dataset_converter_config.pinhole_camera_store_option == "binary":
-                with open(image_abs_path, "rb") as f:
-                    camera_data = f.read()
-            camera_dict[camera_type] = camera_data, camera_extrinsic
-
-    return camera_dict
+    return camera_data_list
 
 
 def _extract_pandaset_lidar(

src/py123d/conversion/datasets/wopd/waymo_sensor_io.py

@@ -28,7 +28,11 @@ def _get_frame_at_iteration(filepath: Path, iteration: int) -> Optional[dataset_
     return frame
 
 
-def load_jpeg_binary_from_file(tf_record_path: Path, iteration: int, pinhole_camera_type: PinholeCameraType) -> bytes:
+def load_jpeg_binary_from_tf_record_file(
+    tf_record_path: Path,
+    iteration: int,
+    pinhole_camera_type: PinholeCameraType,
+) -> bytes:
     frame = _get_frame_at_iteration(tf_record_path, iteration)
     assert frame is not None, f"Frame at iteration {iteration} not found in Waymo file: {tf_record_path}"
 

src/py123d/conversion/datasets/wopd/wopd_converter.py

@@ -17,7 +17,7 @@
     WOPD_LIDAR_TYPES,
 )
 from py123d.conversion.datasets.wopd.waymo_map_utils.wopd_map_utils import convert_wopd_map
-from py123d.conversion.log_writer.abstract_log_writer import AbstractLogWriter, LiDARData
+from py123d.conversion.log_writer.abstract_log_writer import AbstractLogWriter, CameraData, LiDARData
 from py123d.conversion.map_writer.abstract_map_writer import AbstractMapWriter
 from py123d.conversion.registry.box_detection_label_registry import WOPDBoxDetectionLabel
 from py123d.conversion.registry.lidar_index_registry import DefaultLiDARIndex, WOPDLiDARIndex
@@ -379,9 +379,9 @@ def _extract_wopd_box_detections(
 
 def _extract_wopd_cameras(
     frame: dataset_pb2.Frame, dataset_converter_config: DatasetConverterConfig
-) -> Dict[PinholeCameraType, Tuple[Union[str, bytes], StateSE3]]:
+) -> List[CameraData]:
 
-    camera_dict: Dict[PinholeCameraType, Tuple[Union[str, bytes], StateSE3]] = {}
+    camera_data_list: List[CameraData] = []
 
     if dataset_converter_config.include_pinhole_cameras:
 
@@ -404,10 +404,15 @@ def _extract_wopd_cameras(
 
         for image_proto in frame.images:
             camera_type = WOPD_CAMERA_TYPES[image_proto.name]
-            camera_bytes: bytes = image_proto.image
-            camera_dict[camera_type] = camera_bytes, camera_extrinsic[camera_type]
+            camera_data_list.append(
+                CameraData(
+                    camera_type=camera_type,
+                    extrinsic=camera_extrinsic[camera_type],
+                    jpeg_binary=image_proto.image,
+                )
+            )
 
-    return camera_dict
+    return camera_data_list
 
 
 def _extract_wopd_lidars(