|
| 1 | +# coding: utf-8 |
| 2 | +__author__ = 'ZFTurbo: https://kaggle.com/zfturbo' |
| 3 | + |
| 4 | +import numpy as np |
| 5 | +import pandas as pd |
| 6 | +import json |
| 7 | +import time |
| 8 | +from pycocotools.coco import COCO |
| 9 | +from pycocotools.cocoeval import COCOeval |
| 10 | +from multiprocessing import Pool, Process, cpu_count, Manager |
| 11 | +from ensemble_boxes import * |
| 12 | + |
| 13 | + |
| 14 | +def get_coco_annotations_data(): |
| 15 | + file_in = 'instances_val2017.json' |
| 16 | + images = dict() |
| 17 | + with open(file_in) as json_file: |
| 18 | + data = json.load(json_file) |
| 19 | + for i in range(len(data['images'])): |
| 20 | + image_id = data['images'][i]['id'] |
| 21 | + images[image_id] = data['images'][i] |
| 22 | + |
| 23 | + return images |
| 24 | + |
| 25 | + |
| 26 | +def get_coco_score(csv_path): |
| 27 | + images = get_coco_annotations_data() |
| 28 | + s = pd.read_csv(csv_path, dtype={'img_id': np.str, 'label': np.str}) |
| 29 | + |
| 30 | + out = np.zeros((len(s), 7), dtype=np.float64) |
| 31 | + out[:, 0] = s['img_id'] |
| 32 | + ids = s['img_id'].astype(np.int32).values |
| 33 | + x1 = s['x1'].values |
| 34 | + x2 = s['x2'].values |
| 35 | + y1 = s['y1'].values |
| 36 | + y2 = s['y2'].values |
| 37 | + for i in range(len(s)): |
| 38 | + width = images[ids[i]]['width'] |
| 39 | + height = images[ids[i]]['height'] |
| 40 | + out[i, 1] = x1[i] * width |
| 41 | + out[i, 2] = y1[i] * height |
| 42 | + out[i, 3] = (x2[i] - x1[i]) * width |
| 43 | + out[i, 4] = (y2[i] - y1[i]) * height |
| 44 | + out[:, 5] = s['score'].values |
| 45 | + out[:, 6] = s['label'].values |
| 46 | + |
| 47 | + filename = 'instances_val2017.json' |
| 48 | + coco_gt = COCO(filename) |
| 49 | + detections = out |
| 50 | + print(detections.shape) |
| 51 | + print(detections[:5]) |
| 52 | + image_ids = list(set(detections[:, 0])) |
| 53 | + coco_dt = coco_gt.loadRes(detections) |
| 54 | + coco_eval = COCOeval(coco_gt, coco_dt, iouType='bbox') |
| 55 | + coco_eval.params.imgIds = image_ids |
| 56 | + coco_eval.evaluate() |
| 57 | + coco_eval.accumulate() |
| 58 | + coco_eval.summarize() |
| 59 | + coco_metrics = coco_eval.stats |
| 60 | + print(coco_metrics) |
| 61 | + return coco_metrics, detections |
| 62 | + |
| 63 | + |
| 64 | +def process_single_id(id, res_boxes, weights, params): |
| 65 | + run_type = params['run_type'] |
| 66 | + verbose = params['verbose'] |
| 67 | + |
| 68 | + # print('Go for ID: {}'.format(id)) |
| 69 | + boxes_list = [] |
| 70 | + scores_list = [] |
| 71 | + labels_list = [] |
| 72 | + labels_to_use_forward = dict() |
| 73 | + labels_to_use_backward = dict() |
| 74 | + |
| 75 | + for i in range(len(res_boxes[id])): |
| 76 | + boxes = [] |
| 77 | + scores = [] |
| 78 | + labels = [] |
| 79 | + |
| 80 | + dt = res_boxes[id][i] |
| 81 | + |
| 82 | + for j in range(0, len(dt)): |
| 83 | + lbl = dt[j][5] |
| 84 | + scr = float(dt[j][4]) |
| 85 | + box_x1 = float(dt[j][0]) |
| 86 | + box_y1 = float(dt[j][1]) |
| 87 | + box_x2 = float(dt[j][2]) |
| 88 | + box_y2 = float(dt[j][3]) |
| 89 | + |
| 90 | + if box_x1 >= box_x2: |
| 91 | + if verbose: |
| 92 | + print('Problem with box x1 and x2: {}. Skip it'.format(dt[j])) |
| 93 | + continue |
| 94 | + if box_y1 >= box_y2: |
| 95 | + if verbose: |
| 96 | + print('Problem with box y1 and y2: {}. Skip it'.format(dt[j])) |
| 97 | + continue |
| 98 | + if scr <= 0: |
| 99 | + if verbose: |
| 100 | + print('Problem with box score: {}. Skip it'.format(dt[j])) |
| 101 | + continue |
| 102 | + |
| 103 | + boxes.append([box_x1, box_y1, box_x2, box_y2]) |
| 104 | + scores.append(scr) |
| 105 | + if lbl not in labels_to_use_forward: |
| 106 | + cur_point = len(labels_to_use_forward) |
| 107 | + labels_to_use_forward[lbl] = cur_point |
| 108 | + labels_to_use_backward[cur_point] = lbl |
| 109 | + labels.append(labels_to_use_forward[lbl]) |
| 110 | + |
| 111 | + boxes = np.array(boxes, dtype=np.float32) |
| 112 | + scores = np.array(scores, dtype=np.float32) |
| 113 | + labels = np.array(labels, dtype=np.int32) |
| 114 | + |
| 115 | + boxes_list.append(boxes) |
| 116 | + scores_list.append(scores) |
| 117 | + labels_list.append(labels) |
| 118 | + |
| 119 | + # Empty predictions for all models |
| 120 | + if len(boxes_list) == 0: |
| 121 | + return np.array([]), np.array([]), np.array([]) |
| 122 | + |
| 123 | + if run_type == 'wbf': |
| 124 | + merged_boxes, merged_scores, merged_labels = weighted_boxes_fusion(boxes_list, scores_list, labels_list, |
| 125 | + weights=weights, iou_thr=params['intersection_thr'], |
| 126 | + skip_box_thr=params['skip_box_thr'], |
| 127 | + conf_type=params['conf_type']) |
| 128 | + elif run_type == 'nms': |
| 129 | + iou_thr = params['iou_thr'] |
| 130 | + merged_boxes, merged_scores, merged_labels = nms(boxes_list, scores_list, labels_list, weights=weights, iou_thr=iou_thr) |
| 131 | + elif run_type == 'soft-nms': |
| 132 | + iou_thr = params['iou_thr'] |
| 133 | + sigma = params['sigma'] |
| 134 | + thresh = params['thresh'] |
| 135 | + merged_boxes, merged_scores, merged_labels = soft_nms(boxes_list, scores_list, labels_list, |
| 136 | + weights=weights, iou_thr=iou_thr, sigma=sigma, thresh=thresh) |
| 137 | + elif run_type == 'nmw': |
| 138 | + merged_boxes, merged_scores, merged_labels = non_maximum_weighted(boxes_list, scores_list, labels_list, |
| 139 | + weights=weights, iou_thr=params['intersection_thr'], |
| 140 | + skip_box_thr=params['skip_box_thr']) |
| 141 | + |
| 142 | + # print(len(boxes_list), len(merged_boxes)) |
| 143 | + if 'limit_boxes' in params: |
| 144 | + limit_boxes = params['limit_boxes'] |
| 145 | + if len(merged_boxes) > limit_boxes: |
| 146 | + merged_boxes = merged_boxes[:limit_boxes] |
| 147 | + merged_scores = merged_scores[:limit_boxes] |
| 148 | + merged_labels = merged_labels[:limit_boxes] |
| 149 | + |
| 150 | + # Rename labels back |
| 151 | + merged_labels_string = [] |
| 152 | + for m in merged_labels: |
| 153 | + merged_labels_string.append(labels_to_use_backward[m]) |
| 154 | + merged_labels = np.array(merged_labels_string, dtype=np.str) |
| 155 | + |
| 156 | + # Create IDs array |
| 157 | + ids_list = [id] * len(merged_labels) |
| 158 | + |
| 159 | + return merged_boxes.copy(), merged_scores.copy(), merged_labels.copy(), ids_list.copy() |
| 160 | + |
| 161 | + |
| 162 | +def process_part_of_data(proc_number, return_dict, ids_to_use, res_boxes, weights, params): |
| 163 | + print('Start process: {} IDs to proc: {}'.format(proc_number, len(ids_to_use))) |
| 164 | + result = [] |
| 165 | + for id in ids_to_use: |
| 166 | + merged_boxes, merged_scores, merged_labels, ids_list = process_single_id(id, res_boxes, weights, params) |
| 167 | + # print(merged_boxes.shape, merged_scores.shape, merged_labels.shape, len(ids_list)) |
| 168 | + result.append((merged_boxes, merged_scores, merged_labels, ids_list)) |
| 169 | + return_dict[proc_number] = result.copy() |
| 170 | + |
| 171 | + |
| 172 | +def ensemble_predictions(pred_filenames, weights, params): |
| 173 | + verbose = False |
| 174 | + if 'verbose' in params: |
| 175 | + verbose = params['verbose'] |
| 176 | + |
| 177 | + start_time = time.time() |
| 178 | + procs_to_use = max(cpu_count() // 2, 1) |
| 179 | + # procs_to_use = 6 |
| 180 | + print('Use processes: {}'.format(procs_to_use)) |
| 181 | + weights = np.array(weights) |
| 182 | + |
| 183 | + res_boxes = dict() |
| 184 | + ref_ids = None |
| 185 | + for j in range(len(pred_filenames)): |
| 186 | + if weights[j] == 0: |
| 187 | + continue |
| 188 | + print('Read {}...'.format(pred_filenames[j])) |
| 189 | + s = pd.read_csv(pred_filenames[j], dtype={'img_id': np.str, 'label': np.str}) |
| 190 | + s.sort_values('img_id', inplace=True) |
| 191 | + s.reset_index(drop=True, inplace=True) |
| 192 | + ids = s['img_id'].values |
| 193 | + unique_ids = sorted(s['img_id'].unique()) |
| 194 | + if ref_ids is None: |
| 195 | + ref_ids = tuple(unique_ids) |
| 196 | + else: |
| 197 | + if ref_ids != tuple(unique_ids): |
| 198 | + print('Different IDs in ensembled CSVs! {} != {}'.format(len(ref_ids), len(unique_ids))) |
| 199 | + s = s[s['img_id'].isin(ref_ids)] |
| 200 | + s.sort_values('img_id', inplace=True) |
| 201 | + s.reset_index(drop=True, inplace=True) |
| 202 | + ids = s['img_id'].values |
| 203 | + preds = s[['x1', 'y1', 'x2', 'y2', 'score', 'label']].values |
| 204 | + single_res = dict() |
| 205 | + for i in range(len(ids)): |
| 206 | + id = ids[i] |
| 207 | + if id not in single_res: |
| 208 | + single_res[id] = [] |
| 209 | + single_res[id].append(preds[i]) |
| 210 | + for el in single_res: |
| 211 | + if el not in res_boxes: |
| 212 | + res_boxes[el] = [] |
| 213 | + res_boxes[el].append(single_res[el]) |
| 214 | + |
| 215 | + # Reduce weights if needed |
| 216 | + weights = weights[weights != 0] |
| 217 | + |
| 218 | + ids_to_use = sorted(list(res_boxes.keys())) |
| 219 | + manager = Manager() |
| 220 | + return_dict = manager.dict() |
| 221 | + jobs = [] |
| 222 | + for i in range(procs_to_use): |
| 223 | + start = i * len(ids_to_use) // procs_to_use |
| 224 | + end = (i+1) * len(ids_to_use) // procs_to_use |
| 225 | + if i == procs_to_use - 1: |
| 226 | + end = len(ids_to_use) |
| 227 | + p = Process(target=process_part_of_data, args=(i, return_dict, ids_to_use[start:end], res_boxes, weights, params)) |
| 228 | + jobs.append(p) |
| 229 | + p.start() |
| 230 | + |
| 231 | + for i in range(len(jobs)): |
| 232 | + jobs[i].join() |
| 233 | + |
| 234 | + results = [] |
| 235 | + for i in range(len(jobs)): |
| 236 | + results += return_dict[i] |
| 237 | + |
| 238 | + # p = Pool(processes=procs_to_use) |
| 239 | + # results = p.starmap(process_single_id, zip(ids_to_use, repeat(weights), repeat(params))) |
| 240 | + |
| 241 | + all_ids = [] |
| 242 | + all_boxes = [] |
| 243 | + all_scores = [] |
| 244 | + all_labels = [] |
| 245 | + for boxes, scores, labels, ids_list in results: |
| 246 | + if boxes is None: |
| 247 | + continue |
| 248 | + all_boxes.append(boxes) |
| 249 | + all_scores.append(scores) |
| 250 | + all_labels.append(labels) |
| 251 | + all_ids.append(ids_list) |
| 252 | + |
| 253 | + all_ids = np.concatenate(all_ids) |
| 254 | + all_boxes = np.concatenate(all_boxes) |
| 255 | + all_scores = np.concatenate(all_scores) |
| 256 | + all_labels = np.concatenate(all_labels) |
| 257 | + if verbose: |
| 258 | + print(all_ids.shape, all_boxes.shape, all_scores.shape, all_labels.shape) |
| 259 | + |
| 260 | + res = pd.DataFrame(all_ids, columns=['img_id']) |
| 261 | + res['label'] = all_labels |
| 262 | + res['score'] = all_scores |
| 263 | + res['x1'] = all_boxes[:, 0] |
| 264 | + res['x2'] = all_boxes[:, 2] |
| 265 | + res['y1'] = all_boxes[:, 1] |
| 266 | + res['y2'] = all_boxes[:, 3] |
| 267 | + print('Run time: {:.2f}'.format(time.time() - start_time)) |
| 268 | + return res |
| 269 | + |
| 270 | + |
| 271 | +def ensemble(benchmark_csv, weights, params, get_score_init=True): |
| 272 | + if get_score_init: |
| 273 | + for bcsv in benchmark_csv: |
| 274 | + print('Go for {}'.format(bcsv)) |
| 275 | + get_coco_score(bcsv) |
| 276 | + |
| 277 | + ensemble_preds = ensemble_predictions(benchmark_csv, weights, params) |
| 278 | + ensemble_preds.to_csv("ensemble.csv", index=False) |
| 279 | + get_coco_score("ensemble.csv") |
| 280 | + |
| 281 | + |
| 282 | +if __name__ == '__main__': |
| 283 | + if 0: |
| 284 | + params = { |
| 285 | + 'run_type': 'nms', |
| 286 | + 'iou_thr': 0.5, |
| 287 | + 'verbose': True, |
| 288 | + } |
| 289 | + if 0: |
| 290 | + params = { |
| 291 | + 'run_type': 'soft-nms', |
| 292 | + 'iou_thr': 0.5, |
| 293 | + 'thresh': 0.0001, |
| 294 | + 'sigma': 0.1, |
| 295 | + 'verbose': True, |
| 296 | + } |
| 297 | + if 0: |
| 298 | + params = { |
| 299 | + 'run_type': 'nmw', |
| 300 | + 'skip_box_thr': 0.000000001, |
| 301 | + 'intersection_thr': 0.5, |
| 302 | + 'limit_boxes': 30000, |
| 303 | + 'verbose': True, |
| 304 | + } |
| 305 | + |
| 306 | + if 1: |
| 307 | + params = { |
| 308 | + 'run_type': 'wbf', |
| 309 | + 'skip_box_thr': 0.001, |
| 310 | + 'intersection_thr': 0.7, |
| 311 | + 'conf_type': 'avg', |
| 312 | + 'limit_boxes': 30000, |
| 313 | + 'verbose': False, |
| 314 | + } |
| 315 | + |
| 316 | + in_dir = './' |
| 317 | + benchmark_csv = [ |
| 318 | + in_dir + 'EffNetB0-preds.csv', |
| 319 | + in_dir + 'EffNetB0-mirror-preds.csv', |
| 320 | + in_dir + 'EffNetB1-preds.csv', |
| 321 | + in_dir + 'EffNetB1-mirror-preds.csv', |
| 322 | + in_dir + 'EffNetB2-preds.csv', |
| 323 | + in_dir + 'EffNetB2-mirror-preds.csv', |
| 324 | + in_dir + 'EffNetB3-preds.csv', |
| 325 | + in_dir + 'EffNetB3-mirror-preds.csv', |
| 326 | + in_dir + 'EffNetB4-preds.csv', |
| 327 | + in_dir + 'EffNetB4-mirror-preds.csv', |
| 328 | + in_dir + 'EffNetB5-preds.csv', |
| 329 | + in_dir + 'EffNetB5-mirror-preds.csv', |
| 330 | + in_dir + 'EffNetB6-preds.csv', |
| 331 | + in_dir + 'EffNetB6-mirror-preds.csv', |
| 332 | + in_dir + 'EffNetB7-preds.csv', |
| 333 | + in_dir + 'EffNetB7-mirror-preds.csv', |
| 334 | + in_dir + 'DetRS-valid.csv', |
| 335 | + in_dir + 'DetRS-mirror-valid.csv', |
| 336 | + in_dir + 'DetRS_resnet50-valid.csv', |
| 337 | + in_dir + 'DetRS_resnet50-mirror-valid.csv', |
| 338 | + ] |
| 339 | + weights = [0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 5, 5, 7, 7, 9, 9, 8, 8, 5, 5] |
| 340 | + assert(len(benchmark_csv) == len(weights)) |
| 341 | + ensemble(benchmark_csv, weights, params, True) |
| 342 | + |
0 commit comments