import argparse import os # limit the number of cpus used by high performance libraries # os.environ["OMP_NUM_THREADS"] = "8" # os.environ["OPENBLAS_NUM_THREADS"] = "8" # os.environ["MKL_NUM_THREADS"] = "8" # os.environ["VECLIB_MAXIMUM_THREADS"] = "8" # os.environ["NUMEXPR_NUM_THREADS"] = "8" import platform import sys import numpy as np from pathlib import Path import torch import torch.backends.cudnn as cudnn from numpy import random from time import time import pandas as pd FILE = Path(__file__).resolve() ROOT = FILE.parents[0] # yolov5 strongsort root directory WEIGHTS = ROOT / 'weights' if str(ROOT) not in sys.path: sys.path.append(str(ROOT)) # add ROOT to PATH if str(ROOT / 'yolov9') not in sys.path: sys.path.append(str(ROOT / 'yolov9')) # add yolov5 ROOT to PATH if str(ROOT / 'strong_sort') not in sys.path: sys.path.append(str(ROOT / 'strong_sort')) # add strong_sort ROOT to PATH ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative from models.experimental import attempt_load from models.common import DetectMultiBackend from utils.dataloaders import LoadImages, LoadStreams, LoadScreenshots from utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2, increment_path, non_max_suppression, print_args, scale_boxes, strip_optimizer, xyxy2xywh) from utils.torch_utils import select_device, time_sync, smart_inference_mode from utils.plots import Annotator, colors, save_one_box from strong_sort.utils.parser import get_config from strong_sort.strong_sort import StrongSORT VID_FORMATS = 'asf', 'avi', 'gif', 'm4v', 'mkv', 'mov', 'mp4', 'mpeg', 'mpg', 'ts', 'wmv' # include video suffixes def plot_one_box(x, img, color=None, label=None, line_thickness=3): # Plots one bounding box on image img tl = line_thickness or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 1 # line/font thickness c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3])) cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA) if label: tf = max(tl - 1, 1) # font thickness t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0] c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3 cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA) # filled cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA) def convert_to_int(tensor): return tensor.type(torch.int16).item() @smart_inference_mode() def run_strongsort( source='0', data = ROOT / 'data/coco.yaml', # data.yaml path yolo_weights=WEIGHTS / 'yolo.pt', # model.pt path(s), strong_sort_weights=WEIGHTS / 'osnet_x0_25_msmt17.pt', # model.pt path, config_strongsort=ROOT / 'strong_sort/configs/strong_sort.yaml', imgsz=(640, 640), # inference size (height, width) conf_thres=0.25, # confidence threshold iou_thres=0.45, # NMS IOU threshold max_det=1000, # maximum detections per image device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu view_img=False, # show results save_txt=False, # save results to *.txt save_conf=False, # save confidences in --save-txt labels save_crop=False, # save cropped prediction boxes nosave=False, # do not save images/videos classes=None, # filter by class: --class 0, or --class 0 2 3 agnostic_nms=False, # class-agnostic NMS augment=False, # augmented inference visualize=False, # visualize features update=False, # update all models project=ROOT / 'runs/track', # save results to project/name name='exp', # save results to project/name exist_ok=False, # existing project/name ok, do not increment line_thickness=3, # bounding box thickness (pixels) hide_labels=False, # hide labels hide_conf=False, # hide confidences half=False, # use FP16 half-precision inference dnn=False, # use OpenCV DNN for ONNX inference vid_stride=1, # video frame-rate stride ): source = str(source) save_img = not nosave and not source.endswith('.txt') # save inference images is_file = Path(source).suffix[1:] in (VID_FORMATS) is_url = source.lower().startswith(('rtsp://', 'rtmp://', 'http://', 'https://')) webcam = source.isnumeric() or source.endswith('.txt') or (is_url and not is_file) screenshot = source.lower().startswith('screen') if is_url and is_file: source = check_file(source) # download # Directories if not isinstance(yolo_weights, list): # single yolo model exp_name = Path(yolo_weights).stem elif type(yolo_weights) is list and len(yolo_weights) == 1: # single models after --yolo_weights exp_name = Path(yolo_weights[0]).stem yolo_weights = Path(yolo_weights[0]) else: # multiple models after --yolo_weights exp_name = 'ensemble' exp_name = name if name else exp_name + "_" + Path(strong_sort_weights).stem save_dir = increment_path(Path(project) / exp_name, exist_ok=exist_ok) # increment run save_dir = Path(save_dir) (save_dir / 'tracks' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir # Load model device = select_device(device) model = DetectMultiBackend(yolo_weights, device=device, dnn=dnn, data=data, fp16=half) stride, names, pt = model.stride, model.names, model.pt imgsz = check_img_size(imgsz, s=stride) # check image size # Dataloader # Dataloader bs = 1 # batch_size if webcam: view_img = check_imshow(warn=True) dataset = LoadStreams(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) bs = len(dataset) elif screenshot: dataset = LoadScreenshots(source, img_size=imgsz, stride=stride, auto=pt) else: dataset = LoadImages(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) vid_path, vid_writer,txt_path = [None] * bs, [None] * bs, [None] * bs # initialize StrongSORT cfg = get_config() cfg.merge_from_file(config_strongsort) # Create as many strong sort instances as there are video sources strongsort_list = [] for i in range(bs): strongsort_list.append( StrongSORT( strong_sort_weights, device, half, max_dist=cfg.STRONGSORT.MAX_DIST, max_iou_distance=cfg.STRONGSORT.MAX_IOU_DISTANCE, max_age=cfg.STRONGSORT.MAX_AGE, n_init=cfg.STRONGSORT.N_INIT, nn_budget=cfg.STRONGSORT.NN_BUDGET, mc_lambda=cfg.STRONGSORT.MC_LAMBDA, ema_alpha=cfg.STRONGSORT.EMA_ALPHA, ) ) strongsort_list[i].model.warmup() outputs = [None] * bs colors = [[0, 0, 255], [255, 148, 0], [0, 255, 10], [0, 247, 250], [235,0,255]] #250, 247, 0 # Run tracking model.warmup(imgsz=(1 if pt or model.triton else bs, 3, *imgsz)) # warmup seen, windows, dt,sdt = 0, [], (Profile(), Profile(), Profile(), Profile()),[0.0, 0.0, 0.0, 0.0] curr_frames, prev_frames = [None] * bs, [None] * bs frame_counts = [] for frame_idx, (path, im, im0s, vid_cap, s) in enumerate(dataset): # s = '' t1 = time_sync() with dt[0]: im = torch.from_numpy(im).to(model.device) im = im.half() if model.fp16 else im.float() # uint8 to fp16/32 im /= 255 # 0 - 255 to 0.0 - 1.0 if len(im.shape) == 3: im = im[None] # expand for batch dim t2 = time_sync() sdt[0] += t2 - t1 # Inference with dt[1]: visualize = increment_path(save_dir / Path(path).stem, mkdir=True) if visualize else False pred = model(im, augment=augment, visualize=visualize) # pred = pred[0][1] t3 = time_sync() sdt[1] += t3 - t2 # Apply NMS with dt[2]: pred = pred[0][1] if isinstance(pred[0], list) else pred[0] # single model or ensemble pred = non_max_suppression(pred, conf_thres, iou_thres, classes, agnostic_nms, max_det=max_det) sdt[2] += time_sync() - t3 # Second-stage classifier (optional) # pred = utils.general.apply_classifier(pred, classifier_model, im, im0s) counts = {} # Process detections for i, det in enumerate(pred): # detections per image seen += 1 if webcam: # bs >= 1 p, im0, _ = path[i], im0s[i].copy(), dataset.count p = Path(p) # to Path s += f'{i}: ' # txt_file_name = p.name txt_file_name = p.stem + f'_{i}' # Unique text file name # save_path = str(save_dir / p.name) + str(i) # im.jpg, vid.mp4, ... save_path = str(save_dir / p.stem) + f'_{i}' # Unique video file name else: p, im0, _ = path, im0s.copy(), getattr(dataset, 'frame', 0) p = Path(p) # to Path # video file if source.endswith(VID_FORMATS): txt_file_name = p.stem save_path = str(save_dir / p.name) # im.jpg, vid.mp4, ... # folder with imgs else: txt_file_name = p.parent.name # get folder name containing current img save_path = str(save_dir / p.parent.name) # im.jpg, vid.mp4, ... curr_frames[i] = im0 txt_path = str(save_dir / 'tracks' / txt_file_name) # im.txt s += '%gx%g ' % im.shape[2:] # print string gn = torch.tensor(im0.shape)[[1, 0, 1, 0]] # normalization gain whwh imc = im0.copy() if save_crop else im0 # for save_crop annotator = Annotator(im0, line_width=line_thickness, example=str(names)) if cfg.STRONGSORT.ECC: # camera motion compensation strongsort_list[i].tracker.camera_update(prev_frames[i], curr_frames[i]) if det is not None and len(det): # Rescale boxes from img_size to im0 size det[:, :4] = scale_boxes(im.shape[2:], det[:, :4], im0.shape).round() # Print results for c in det[:, -1].unique(): n = (det[:, -1] == c).sum() # detections per class s += f"{n} {names[int(c)]}{'s' * (n > 1)}, " # add to string counts[names[int(c)]] = n xywhs = xyxy2xywh(det[:, 0:4]) confs = det[:, 4] clss = det[:, 5] # pass detections to strongsort t4 = time_sync() outputs[i] = strongsort_list[i].update(xywhs.cpu(), confs.cpu(), clss.cpu(), im0) t5 = time_sync() sdt[3] += t5 - t4 # Write results for j, (output, conf) in enumerate(zip(outputs[i], confs)): xyxy = output[0:4] id = output[4] cls = output[5] label = names[int(cls)] # for *xyxy, conf, cls in reversed(det): if save_txt: # Write to file xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh # line = (id , cls, *xywh, conf) if save_conf else (cls, *xywh) # label format line = ( int(p.stem), frame_idx, id , cls, *xywh, conf) if save_conf else ( p.stem, frame_idx, cls, *xywh) # label format with open(txt_path + '.txt', 'a') as file: file.write(('%g ' * len(line) + '\n') % line) if save_img or save_crop or view_img: # Add bbox to image c = int(cls) # integer class label = None if hide_labels else ( str(id) + ' ' + names[c] if hide_conf else f' { id } {names[c]} {conf:.2f}') plot_one_box(xyxy, im0, label=label, color=colors[int(cls)], line_thickness=2) if save_crop: save_one_box(xyxy, imc, file=save_dir / 'crops' / names[c] / f'{p.stem}.jpg', BGR=True) frame_counts.append({'frame': frame_idx, 'counts': counts.copy()}) # # draw boxes for visualization # if len(outputs[i]) > 0: # for j, (output, conf) in enumerate(zip(outputs[i], confs)): # bboxes = output[0:4] # id = output[4] # cls = output[5] # if save_txt: # # to MOT format # bbox_left = output[0] # bbox_top = output[1] # bbox_w = output[2] - output[0] # bbox_h = output[3] - output[1] # # format video_name frame id xmin ymin width height score class # with open(txt_path + '.txt', 'a') as file: # file.write(f'{p.stem} {frame_idx} {id} {bbox_left} {bbox_top} {bbox_w} {bbox_h} {conf:.2f} {cls}\n') # if save_img or save_crop or view_img: # Add bbox to image # c = int(cls) # integer class # id = int(id) # integer id # label = None if hide_labels else (names[c] if hide_conf else f'{names[c]} {conf:.2f}') # plot_one_box(bboxes, im0, label=label, color=colors[int(cls)], line_thickness=2) # if save_crop: # txt_file_name = txt_file_name if (isinstance(path, list) and len(path) > 1) else '' # save_one_box(bboxes, imc, file=save_dir / 'crops' / txt_file_name / names[c] / f'{id}' / f'{p.stem}.jpg', BGR=True) print(f'{s}Done. YOLO:({t3 - t2:.3f}s), StrongSORT:({t5 - t4:.3f}s)') else: strongsort_list[i].increment_ages() print('No detections') # Stream results im0 = annotator.result() if view_img: if platform.system() == 'Linux' and p not in windows: windows.append(p) cv2.namedWindow(str(p), cv2.WINDOW_NORMAL | cv2.WINDOW_KEEPRATIO) # allow window resize (Linux) cv2.resizeWindow(str(p), im0.shape[1], im0.shape[0]) cv2.imshow(str(p), im0) cv2.waitKey(1) # 1 millisecond # Save results (image with detections) if save_img: if dataset.mode == 'image': cv2.imwrite(save_path, im0) else: # 'video' or 'stream' if vid_path[i] != save_path: # new video vid_path[i] = save_path if isinstance(vid_writer[i], cv2.VideoWriter): vid_writer[i].release() # release previous video writer if vid_cap: # video fps = vid_cap.get(cv2.CAP_PROP_FPS) w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) else: # stream fps, w, h = 30, im0.shape[1], im0.shape[0] save_path = str(Path(save_path).with_suffix('.mp4')) # force *.mp4 suffix on results videos vid_writer[i] = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc('m','p','4','v'), fps, (w, h)) vid_writer[i].write(im0) prev_frames[i] = curr_frames[i] # Print time (inference-only) LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") flattened_counts = [ {'frame': entry['frame'], 'label': label, 'count': count} for entry in frame_counts for label, count in entry['counts'].items() ] frame_counts_df = pd.DataFrame(flattened_counts) frame_counts_df['count'] = frame_counts_df['count'].apply(convert_to_int) counts_df = None # Print results LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape, %.1fms StrongSORT' % tuple(1E3 * x / seen for x in sdt)) if save_txt or save_img: s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}") if update: strip_optimizer(yolo_weights[0]) # update model (to fix SourceChangeWarning) return save_path, counts_df, frame_counts_df def parse_opt(): parser = argparse.ArgumentParser() parser.add_argument('--yolo-weights', nargs='+', type=str, default=WEIGHTS / 'yolov9.pt', help='model.pt path(s)') parser.add_argument('--strong-sort-weights', type=str, default=WEIGHTS / 'osnet_x0_25_msmt17.pt') parser.add_argument('--config-strongsort', type=str, default='strong_sort/configs/strong_sort.yaml') parser.add_argument('--source', type=str, default='0', help='file/dir/URL/glob, 0 for webcam') parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='(optional) dataset.yaml path') parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640], help='inference size h,w') parser.add_argument('--conf-thres', type=float, default=0.5, help='confidence threshold') parser.add_argument('--iou-thres', type=float, default=0.5, help='NMS IoU threshold') parser.add_argument('--max-det', type=int, default=1000, help='maximum detections per image') parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') parser.add_argument('--view-img', action='store_true', help='show results') parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels') parser.add_argument('--save-crop', action='store_true', help='save cropped prediction boxes') parser.add_argument('--nosave', action='store_true', help='do not save images/videos') # class 0 is person, 1 is bycicle, 2 is car... 79 is oven parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --classes 0, or --classes 0 2 3') parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS') parser.add_argument('--augment', action='store_true', help='augmented inference') parser.add_argument('--visualize', action='store_true', help='visualize features') parser.add_argument('--update', action='store_true', help='update all models') parser.add_argument('--project', default=ROOT / 'runs/track', help='save results to project/name') parser.add_argument('--name', default='exp', help='save results to project/name') parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') parser.add_argument('--line-thickness', default=3, type=int, help='bounding box thickness (pixels)') parser.add_argument('--hide-labels', default=False, action='store_true', help='hide labels') parser.add_argument('--hide-conf', default=False, action='store_true', help='hide confidences') parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference') parser.add_argument('--vid-stride', type=int, default=1, help='video frame-rate stride') parser.add_argument('--dnn', action='store_true', help='use OpenCV DNN for ONNX inference') opt = parser.parse_args() opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand return opt def main(opt): # check_requirements(requirements=ROOT / 'requirements.txt', exclude=('tensorboard', 'thop')) run_strongsort(**vars(opt)) if __name__ == "__main__": opt = parse_opt() main(opt)