Basic Visualization
Example visualization programs here are based on the parsed dataset.
Please refer to V2X-Sim 2.0 tutorial if you are looking for visualizations for the raw V2X-Sim dataset.
Detection dataset
To visualize voxelized point cloud with ground-truth bounding box:
import matplotlib.pyplot as plt
from coperception.datasets import V2XSimDet, V2XSimSeg
from coperception.configs import Config, ConfigGlobal
from coperception.utils.obj_util import *
split = 'train'
config = Config(binary=True, split=split, use_vis=True)
config_global = ConfigGlobal(binary=True, split=split)
data_path = f'/path/to/created/det/dataset/{split}/agent0'
v2x_det_dataset = V2XSimDet(
dataset_roots=[data_path], split=split, config_global=config_global,
config=config, val=True, bound='both')
padded_voxel_points, padded_voxel_points_teacher_det, label_one_hot, reg_target, reg_loss_mask, anchors_map, \
vis_maps, gt_max_iou, filename, target_agent_id, num_sensor, trans_matrix = v2x_det_dataset[0][0]
plt.imshow(np.max(padded_voxel_points.reshape(256, 256, 13), axis=2), alpha=1.0, zorder=12)
# draw ground-truth bounding box
gt_max_iou_idx = gt_max_iou[0]['gt_box']
for p in range(v2x_det_dataset.pred_len):
for k in range(len(gt_max_iou_idx)):
anchor = anchors_map[tuple(gt_max_iou_idx[k][:-1])]
encode_box = reg_target[tuple(gt_max_iou_idx[k][:-1]) + (p,)]
decode_box = bev_box_decode_np(encode_box, anchor)
decode_corner = center_to_corner_box2d(
np.asarray([anchor[:2]]),
np.asarray([anchor[2:4]]),
np.asarray([anchor[4:]]))[0]
corners = coor_to_vis(decode_corner, v2x_det_dataset.area_extents, v2x_det_dataset.voxel_size)
c_x, c_y = np.mean(corners, axis=0)
corners = np.concatenate([corners, corners[[0]]])
plt.plot(corners[:, 0], corners[:, 1], c='g', linewidth=2.0, zorder=20)
plt.scatter(c_x, c_y, s=3, c='g', zorder=20)
plt.plot([c_x, (corners[1][0] + corners[0][0]) / 2.], [c_y, (corners[1][1] + corners[0][1]) / 2.],
linewidth=2.0, c='g', zorder=20)
occupy = np.max(vis_maps, axis=-1)
m = np.stack([occupy, occupy, occupy], axis=-1)
m[m > 0] = 0.99
occupy = (m * 255).astype(np.uint8)
free = np.min(vis_maps, axis=-1)
m = np.stack([free, free, free], axis=-1)
m[m < 0] = 0.5
free = (m * 255).astype(np.uint8)
Segmentation dataset
To visulize voxelized point cloud:
import matplotlib.pyplot as plt
from coperception.datasets import V2XSimDet, V2XSimSeg
from coperception.configs import Config, ConfigGlobal
from coperception.utils.obj_util import *
split = 'train'
config = Config(binary=True, split=split, use_vis=True)
data_path = f'/path/to/created/seg/dataset/{split}/agent0'
v2x_seg_dataset = V2XSimSeg(dataset_roots=[data_path],
split=split, config=config, val=True, kd_flag=True, bound='both', no_cross_road=False)
padded_voxel_points, padded_voxel_points_teacher, seg_bev = v2x_seg_dataset[0][0]
padded_voxel_points = padded_voxel_points.cpu().detach().numpy()
plt.imshow(np.max(padded_voxel_points.reshape(256, 256, 13), axis=2), alpha=1.0, zorder=12)
To show ground-truth visualization:
import matplotlib.pyplot as plt
from coperception.datasets import V2XSimDet, V2XSimSeg
from coperception.configs import Config, ConfigGlobal
from coperception.utils.obj_util import *
split = 'train'
config = Config(binary=True, split=split, use_vis=True)
data_path = f'/path/to/created/seg/dataset/{split}/agent0'
v2x_seg_dataset = V2XSimSeg(dataset_roots=[data_path],
split=split, config=config, val=True, kd_flag=True, bound='both', no_cross_road=False)
padded_voxel_points, padded_voxel_points_teacher, seg_bev = v2x_seg_dataset[0][0]
padded_voxel_points = padded_voxel_points.cpu().detach().numpy()
gt_image = np.zeros(shape=(256, 256, 3), dtype=np.dtype("uint8"))
# map colors according to our scheme
for key, value in config.class_to_rgb.items():
gt_image[np.where(seg_bev == key)] = value
plt.imshow(gt_image)
The color scheme is defined in class_to_rgb in:
The config class
Source code in
class Config(object):
"""The config class"""
def __init__(
self,
split,
binary=True,
only_det=True,
code_type="faf",
loss_type="faf_loss",
root="",
is_cross_road=False,
use_vis=False,
):
# for segmentaion task only
# =========================
self.num_class = 8
self.in_channels = 13
self.nepoch = 10
self.class_to_rgb = {
0: [255, 255, 255], # Unlabeled
1: [71, 141, 230], # Vehicles
2: [122, 217, 209], # Sidewalk
3: [145, 171, 100], # Ground / Terrain
4: [231, 136, 101], # Road / Traffic light / Pole
5: [142, 80, 204], # Buildings
6: [224, 8, 50], # Pedestrian
7: [106, 142, 34] # Vegetation
# 7: [102, 102, 156], # Walls
# 0: [55, 90, 80], # Other
}
# Remap pixel values given by carla
self.classes_remap = {
0: 0, # Unlabeled (so that we don't forget this class)
10: 1, # Vehicles
8: 2, # Sidewalk
14: 3, # Ground (non-drivable)
22: 3, # Terrain (non-drivable)
7: 4, # Road
6: 4, # Road line
18: 4, # Traffic light
5: 4, # Pole
1: 5, # Building
4: 6, # Pedestrian
9: 7, # Vegetation
}
self.class_idx_to_name = {
0: "Unlabeled",
1: "Vehicles",
2: "Sidewalk",
3: "Ground & Terrain",
4: "Road",
5: "Buildings",
6: "Pedestrian",
7: "Vegetation",
}
# =========================
self.device = None
self.split = split
self.binary = binary
self.only_det = only_det
self.code_type = code_type
self.loss_type = loss_type # corner_loss faf_loss
# The specifications for BEV maps
self.voxel_size = (0.25, 0.25, 0.4)
#self.voxel_size = (0.375, 0.375, 0.4)
self.area_extents = (
np.array([[-32.0, 32.0], [-32.0, 32.0], [-8.0, -3.0]])
#np.array([[-48.0, 48.0], [-48.0, 48.0], [-8.0, -3.0]])
if is_cross_road
else np.array([[-32.0, 32.0], [-32.0, 32.0], [-3.0, 2.0]])
#else np.array([[-48.0, 48.0], [-48.0, 48.0], [-3.0, 2.0]])
)
self.is_cross_road = is_cross_road
self.past_frame_skip = 3 # when generating the BEV maps, how many history frames need to be skipped
self.future_frame_skip = (
0 # when generating the BEV maps, how many future frames need to be skipped
)
self.num_past_frames_for_bev_seq = (
1 # the number of past frames for BEV map sequence
)
self.num_past_pcs = 1 # duplicate self.num_past_frames_for_bev_seq
self.map_dims = [
math.ceil(
(self.area_extents[0][1] - self.area_extents[0][0]) / self.voxel_size[0]
),
math.ceil(
(self.area_extents[1][1] - self.area_extents[1][0]) / self.voxel_size[1]
),
math.ceil(
(self.area_extents[2][1] - self.area_extents[2][0]) / self.voxel_size[2]
),
]
self.only_det = True
self.root = root
# debug Data:
self.code_type = "faf"
self.pred_type = "motion"
# debug Loss
self.loss_type = "corner_loss"
# debug MGDA
self.MGDA = False
# debug when2com
self.MIMO = True
# debug Motion Classification
self.motion_state = False
self.static_thre = 0.2 # speed lower bound
# debug use_vis
self.use_vis = use_vis
self.use_map = False
# The specifications for object detection encode
if self.code_type in ["corner_1", "corner_2"]:
self.box_code_size = 8 # (\delta{x1},\delta{y1},\delta{x2},\delta{y2},\delta{x3},\delta{y3},\delta{x4},\delta{y4})
elif self.code_type in ["corner_3"]:
self.box_code_size = 10
elif self.code_type[0] == "f":
self.box_code_size = 6 # (x,y,w,h,sin,cos)
else:
print(code_type, " code type is not implemented yet!")
exit()
self.pred_len = (
1 # the number of frames for prediction, including the current frame
)
# anchor size: (w,h,angle) (according to nuscenes w < h)
if not self.binary:
self.anchor_size = np.asarray(
[
[2.0, 4.0, 0],
[2.0, 4.0, math.pi / 2.0],
[1.0, 1.0, 0],
[1.0, 2.0, 0.0],
[1.0, 2.0, math.pi / 2.0],
[3.0, 12.0, 0.0],
[3.0, 12.0, math.pi / 2.0],
]
)
else:
self.anchor_size = np.asarray(
[
[2.0, 4.0, 0],
[2.0, 4.0, math.pi / 2.0],
[2.0, 4.0, -math.pi / 4.0],
[3.0, 12.0, 0],
[3.0, 12.0, math.pi / 2.0],
[3.0, 12.0, -math.pi / 4.0],
]
)
self.category_threshold = [0.4, 0.4, 0.25, 0.25, 0.4]
self.class_map = {
"vehicle.car": 1,
"vehicle.emergency.police": 1,
"vehicle.bicycle": 3,
"vehicle.motorcycle": 3,
"vehicle.bus.rigid": 2,
}
if self.binary:
self.category_num = 2
else:
self.category_num = len(self.category_threshold)
self.print_feq = 100
if self.split == "train":
self.num_keyframe_skipped = (
0 # The number of keyframes we will skip when dumping the data
)
self.nsweeps_back = 1 # Number of frames back to the history (including the current timestamp)
self.nsweeps_forward = 0 # Number of frames into the future (does not include the current timestamp)
self.skip_frame = (
0 # The number of frames skipped for the adjacent sequence
)
self.num_adj_seqs = (
1 # number of adjacent sequences, among which the time gap is \delta t
)
else:
self.num_keyframe_skipped = 0
self.nsweeps_back = 1 # Setting this to 30 (for training) or 25 (for testing) allows conducting ablation studies on frame numbers
self.nsweeps_forward = 0
self.skip_frame = 0
self.num_adj_seqs = 1
Other example basic visualization programs are provided in coperception/tools/visualization/v2xsim_vistool_basic