From 85ccd91a7ee6d3b72102a6144cc7e690564a0a5e Mon Sep 17 00:00:00 2001 From: inter Date: Sun, 21 Sep 2025 20:18:36 +0800 Subject: [PATCH] Add File --- pcdet/datasets/once/once_eval/evaluation.py | 420 ++++++++++++++++++++ 1 file changed, 420 insertions(+) create mode 100644 pcdet/datasets/once/once_eval/evaluation.py diff --git a/pcdet/datasets/once/once_eval/evaluation.py b/pcdet/datasets/once/once_eval/evaluation.py new file mode 100644 index 0000000..d890e93 --- /dev/null +++ b/pcdet/datasets/once/once_eval/evaluation.py @@ -0,0 +1,420 @@ +""" +Evaluation Server +Written by Jiageng Mao +""" + +import numpy as np +import numba + +from .iou_utils import rotate_iou_gpu_eval +from .eval_utils import compute_split_parts, overall_filter, distance_filter, overall_distance_filter + +iou_threshold_dict = { + 'Car': 0.7, + 'Bus': 0.7, + 'Truck': 0.7, + 'Pedestrian': 0.3, + 'Cyclist': 0.5 +} + +superclass_iou_threshold_dict = { + 'Vehicle': 0.7, + 'Pedestrian': 0.3, + 'Cyclist': 0.5 +} + +def get_evaluation_results(gt_annos, pred_annos, classes, + use_superclass=True, + iou_thresholds=None, + num_pr_points=50, + difficulty_mode='Overall&Distance', + ap_with_heading=True, + num_parts=100, + print_ok=False + ): + + if iou_thresholds is None: + if use_superclass: + iou_thresholds = superclass_iou_threshold_dict + else: + iou_thresholds = iou_threshold_dict + + assert len(gt_annos) == len(pred_annos), "the number of GT must match predictions" + assert difficulty_mode in ['Overall&Distance', 'Overall', 'Distance'], "difficulty mode is not supported" + if use_superclass: + if ('Car' in classes) or ('Bus' in classes) or ('Truck' in classes): + assert ('Car' in classes) and ('Bus' in classes) and ('Truck' in classes), "Car/Bus/Truck must all exist for vehicle detection" + classes = [cls_name for cls_name in classes if cls_name not in ['Car', 'Bus', 'Truck']] + classes.insert(0, 'Vehicle') + + num_samples = len(gt_annos) + split_parts = compute_split_parts(num_samples, num_parts) + ious = compute_iou3d(gt_annos, pred_annos, split_parts, with_heading=ap_with_heading) + + num_classes = len(classes) + if difficulty_mode == 'Distance': + num_difficulties = 3 + difficulty_types = ['0-30m', '30-50m', '50m-inf'] + elif difficulty_mode == 'Overall': + num_difficulties = 1 + difficulty_types = ['overall'] + elif difficulty_mode == 'Overall&Distance': + num_difficulties = 4 + difficulty_types = ['overall', '0-30m', '30-50m', '50m-inf'] + else: + raise NotImplementedError + + precision = np.zeros([num_classes, num_difficulties, num_pr_points+1]) + recall = np.zeros([num_classes, num_difficulties, num_pr_points+1]) + + for cls_idx, cur_class in enumerate(classes): + iou_threshold = iou_thresholds[cur_class] + for diff_idx in range(num_difficulties): + ### filter data & determine score thresholds on p-r curve ### + accum_all_scores, gt_flags, pred_flags = [], [], [] + num_valid_gt = 0 + for sample_idx in range(num_samples): + gt_anno = gt_annos[sample_idx] + pred_anno = pred_annos[sample_idx] + pred_score = pred_anno['score'] + iou = ious[sample_idx] + gt_flag, pred_flag = filter_data(gt_anno, pred_anno, difficulty_mode, + difficulty_level=diff_idx, class_name=cur_class, use_superclass=use_superclass) + gt_flags.append(gt_flag) + pred_flags.append(pred_flag) + num_valid_gt += sum(gt_flag == 0) + accum_scores = accumulate_scores(iou, pred_score, gt_flag, pred_flag, + iou_threshold=iou_threshold) + accum_all_scores.append(accum_scores) + all_scores = np.concatenate(accum_all_scores, axis=0) + thresholds = get_thresholds(all_scores, num_valid_gt, num_pr_points=num_pr_points) + + ### compute tp/fp/fn ### + confusion_matrix = np.zeros([len(thresholds), 3]) # only record tp/fp/fn + for sample_idx in range(num_samples): + pred_score = pred_annos[sample_idx]['score'] + iou = ious[sample_idx] + gt_flag, pred_flag = gt_flags[sample_idx], pred_flags[sample_idx] + for th_idx, score_th in enumerate(thresholds): + tp, fp, fn = compute_statistics(iou, pred_score, gt_flag, pred_flag, + score_threshold=score_th, iou_threshold=iou_threshold) + confusion_matrix[th_idx, 0] += tp + confusion_matrix[th_idx, 1] += fp + confusion_matrix[th_idx, 2] += fn + + ### draw p-r curve ### + for th_idx in range(len(thresholds)): + recall[cls_idx, diff_idx, th_idx] = confusion_matrix[th_idx, 0] / \ + (confusion_matrix[th_idx, 0] + confusion_matrix[th_idx, 2]) + precision[cls_idx, diff_idx, th_idx] = confusion_matrix[th_idx, 0] / \ + (confusion_matrix[th_idx, 0] + confusion_matrix[th_idx, 1]) + + for th_idx in range(len(thresholds)): + precision[cls_idx, diff_idx, th_idx] = np.max( + precision[cls_idx, diff_idx, th_idx:], axis=-1) + recall[cls_idx, diff_idx, th_idx] = np.max( + recall[cls_idx, diff_idx, th_idx:], axis=-1) + + AP = 0 + for i in range(1, precision.shape[-1]): + AP += precision[..., i] + AP = AP / num_pr_points * 100 + + ret_dict = {} + + ret_str = "\n|AP@%-9s|" % (str(num_pr_points)) + for diff_type in difficulty_types: + ret_str += '%-12s|' % diff_type + ret_str += '\n' + for cls_idx, cur_class in enumerate(classes): + ret_str += "|%-12s|" % cur_class + for diff_idx in range(num_difficulties): + diff_type = difficulty_types[diff_idx] + key = 'AP_' + cur_class + '/' + diff_type + ap_score = AP[cls_idx,diff_idx] + ret_dict[key] = ap_score + ret_str += "%-12.2f|" % ap_score + ret_str += "\n" + mAP = np.mean(AP, axis=0) + ret_str += "|%-12s|" % 'mAP' + for diff_idx in range(num_difficulties): + diff_type = difficulty_types[diff_idx] + key = 'AP_mean' + '/' + diff_type + ap_score = mAP[diff_idx] + ret_dict[key] = ap_score + ret_str += "%-12.2f|" % ap_score + ret_str += "\n" + + if print_ok: + print(ret_str) + return ret_str, ret_dict + +@numba.jit(nopython=True) +def get_thresholds(scores, num_gt, num_pr_points): + eps = 1e-6 + scores.sort() + scores = scores[::-1] + recall_level = 0 + thresholds = [] + for i, score in enumerate(scores): + l_recall = (i + 1) / num_gt + if i < (len(scores) - 1): + r_recall = (i + 2) / num_gt + else: + r_recall = l_recall + if (r_recall + l_recall < 2 * recall_level) and i < (len(scores) - 1): + continue + thresholds.append(score) + recall_level += 1 / num_pr_points + # avoid numerical errors + # while r_recall + l_recall >= 2 * recall_level: + while r_recall + l_recall + eps > 2 * recall_level: + thresholds.append(score) + recall_level += 1 / num_pr_points + return thresholds + +@numba.jit(nopython=True) +def accumulate_scores(iou, pred_scores, gt_flag, pred_flag, iou_threshold): + num_gt = iou.shape[0] + num_pred = iou.shape[1] + assigned = np.full(num_pred, False) + accum_scores = np.zeros(num_gt) + accum_idx = 0 + for i in range(num_gt): + if gt_flag[i] == -1: # not the same class + continue + det_idx = -1 + detected_score = -1 + for j in range(num_pred): + if pred_flag[j] == -1: # not the same class + continue + if assigned[j]: + continue + iou_ij = iou[i, j] + pred_score = pred_scores[j] + if (iou_ij > iou_threshold) and (pred_score > detected_score): + det_idx = j + detected_score = pred_score + + if (detected_score == -1) and (gt_flag[i] == 0): # false negative + pass + elif (detected_score != -1) and (gt_flag[i] == 1 or pred_flag[det_idx] == 1): # ignore + assigned[det_idx] = True + elif detected_score != -1: # true positive + accum_scores[accum_idx] = pred_scores[det_idx] + accum_idx += 1 + assigned[det_idx] = True + + return accum_scores[:accum_idx] + +@numba.jit(nopython=True) +def compute_statistics(iou, pred_scores, gt_flag, pred_flag, score_threshold, iou_threshold): + num_gt = iou.shape[0] + num_pred = iou.shape[1] + assigned = np.full(num_pred, False) + under_threshold = pred_scores < score_threshold + + tp, fp, fn = 0, 0, 0 + for i in range(num_gt): + if gt_flag[i] == -1: # different classes + continue + det_idx = -1 + detected = False + best_matched_iou = 0 + gt_assigned_to_ignore = False + + for j in range(num_pred): + if pred_flag[j] == -1: # different classes + continue + if assigned[j]: # already assigned to other GT + continue + if under_threshold[j]: # compute only boxes above threshold + continue + iou_ij = iou[i, j] + if (iou_ij > iou_threshold) and (iou_ij > best_matched_iou or gt_assigned_to_ignore) and pred_flag[j] == 0: + best_matched_iou = iou_ij + det_idx = j + detected = True + gt_assigned_to_ignore = False + elif (iou_ij > iou_threshold) and (not detected) and pred_flag[j] == 1: + det_idx = j + detected = True + gt_assigned_to_ignore = True + + if (not detected) and gt_flag[i] == 0: # false negative + fn += 1 + elif detected and (gt_flag[i] == 1 or pred_flag[det_idx] == 1): # ignore + assigned[det_idx] = True + elif detected: # true positive + tp += 1 + assigned[det_idx] = True + + for j in range(num_pred): + if not (assigned[j] or pred_flag[j] == -1 or pred_flag[j] == 1 or under_threshold[j]): + fp += 1 + + return tp, fp, fn + +def filter_data(gt_anno, pred_anno, difficulty_mode, difficulty_level, class_name, use_superclass): + """ + Filter data by class name and difficulty + + Args: + gt_anno: + pred_anno: + difficulty_mode: + difficulty_level: + class_name: + + Returns: + gt_flags/pred_flags: + 1 : same class but ignored with different difficulty levels + 0 : accepted + -1 : rejected with different classes + """ + num_gt = len(gt_anno['name']) + gt_flag = np.zeros(num_gt, dtype=np.int64) + if use_superclass: + if class_name == 'Vehicle': + reject = np.logical_or(gt_anno['name']=='Pedestrian', gt_anno['name']=='Cyclist') + else: + reject = gt_anno['name'] != class_name + else: + reject = gt_anno['name'] != class_name + gt_flag[reject] = -1 + num_pred = len(pred_anno['name']) + pred_flag = np.zeros(num_pred, dtype=np.int64) + if use_superclass: + if class_name == 'Vehicle': + reject = np.logical_or(pred_anno['name']=='Pedestrian', pred_anno['name']=='Cyclist') + else: + reject = pred_anno['name'] != class_name + else: + reject = pred_anno['name'] != class_name + pred_flag[reject] = -1 + + if difficulty_mode == 'Overall': + ignore = overall_filter(gt_anno['boxes_3d']) + gt_flag[ignore] = 1 + ignore = overall_filter(pred_anno['boxes_3d']) + pred_flag[ignore] = 1 + elif difficulty_mode == 'Distance': + ignore = distance_filter(gt_anno['boxes_3d'], difficulty_level) + gt_flag[ignore] = 1 + ignore = distance_filter(pred_anno['boxes_3d'], difficulty_level) + pred_flag[ignore] = 1 + elif difficulty_mode == 'Overall&Distance': + ignore = overall_distance_filter(gt_anno['boxes_3d'], difficulty_level) + gt_flag[ignore] = 1 + ignore = overall_distance_filter(pred_anno['boxes_3d'], difficulty_level) + pred_flag[ignore] = 1 + else: + raise NotImplementedError + + return gt_flag, pred_flag + +def iou3d_kernel(gt_boxes, pred_boxes): + """ + Core iou3d computation (with cuda) + + Args: + gt_boxes: [N, 7] (x, y, z, w, l, h, rot) in Lidar coordinates + pred_boxes: [M, 7] + + Returns: + iou3d: [N, M] + """ + intersection_2d = rotate_iou_gpu_eval(gt_boxes[:, [0, 1, 3, 4, 6]], pred_boxes[:, [0, 1, 3, 4, 6]], criterion=2) + gt_max_h = gt_boxes[:, [2]] + gt_boxes[:, [5]] * 0.5 + gt_min_h = gt_boxes[:, [2]] - gt_boxes[:, [5]] * 0.5 + pred_max_h = pred_boxes[:, [2]] + pred_boxes[:, [5]] * 0.5 + pred_min_h = pred_boxes[:, [2]] - pred_boxes[:, [5]] * 0.5 + max_of_min = np.maximum(gt_min_h, pred_min_h.T) + min_of_max = np.minimum(gt_max_h, pred_max_h.T) + inter_h = min_of_max - max_of_min + inter_h[inter_h <= 0] = 0 + #inter_h[intersection_2d <= 0] = 0 + intersection_3d = intersection_2d * inter_h + gt_vol = gt_boxes[:, [3]] * gt_boxes[:, [4]] * gt_boxes[:, [5]] + pred_vol = pred_boxes[:, [3]] * pred_boxes[:, [4]] * pred_boxes[:, [5]] + union_3d = gt_vol + pred_vol.T - intersection_3d + #eps = 1e-6 + #union_3d[union_3d= np.pi] = reverse_diff_rot[diff_rot >= np.pi] # constrain to [0-pi] + iou3d[diff_rot > np.pi/2] = 0 # unmatched if diff_rot > 90 + return iou3d + +def compute_iou3d(gt_annos, pred_annos, split_parts, with_heading): + """ + Compute iou3d of all samples by parts + + Args: + with_heading: filter with heading + gt_annos: list of dicts for each sample + pred_annos: + split_parts: for part-based iou computation + + Returns: + ious: list of iou arrays for each sample + """ + gt_num_per_sample = np.stack([len(anno["name"]) for anno in gt_annos], 0) + pred_num_per_sample = np.stack([len(anno["name"]) for anno in pred_annos], 0) + ious = [] + sample_idx = 0 + for num_part_samples in split_parts: + gt_annos_part = gt_annos[sample_idx:sample_idx + num_part_samples] + pred_annos_part = pred_annos[sample_idx:sample_idx + num_part_samples] + + gt_boxes = np.concatenate([anno["boxes_3d"] for anno in gt_annos_part], 0) + pred_boxes = np.concatenate([anno["boxes_3d"] for anno in pred_annos_part], 0) + + if with_heading: + iou3d_part = iou3d_kernel_with_heading(gt_boxes, pred_boxes) + else: + iou3d_part = iou3d_kernel(gt_boxes, pred_boxes) + + gt_num_idx, pred_num_idx = 0, 0 + for idx in range(num_part_samples): + gt_box_num = gt_num_per_sample[sample_idx + idx] + pred_box_num = pred_num_per_sample[sample_idx + idx] + ious.append(iou3d_part[gt_num_idx: gt_num_idx + gt_box_num, pred_num_idx: pred_num_idx+pred_box_num]) + gt_num_idx += gt_box_num + pred_num_idx += pred_box_num + sample_idx += num_part_samples + return ious \ No newline at end of file