From 5d27e4f4f4d3605f1dd557b24c630994a5f7b748 Mon Sep 17 00:00:00 2001 From: rentainhe <596106517@qq.com> Date: Sat, 31 Aug 2024 12:13:53 +0800 Subject: [PATCH 1/5] refine assert info typo --- grounded_sam2_florence2_image_demo.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/grounded_sam2_florence2_image_demo.py b/grounded_sam2_florence2_image_demo.py index 28a3507..3362807 100644 --- a/grounded_sam2_florence2_image_demo.py +++ b/grounded_sam2_florence2_image_demo.py @@ -109,7 +109,7 @@ def object_detection_and_segmentation( text_input=None, output_dir=OUTPUT_DIR ): - assert text_input is None, "Text input should not be none when calling object detection pipeline." + assert text_input is None, "Text input should be None when calling object detection pipeline." # run florence-2 object detection in demo image = Image.open(image_path).convert("RGB") results = run_florence2(task_prompt, text_input, florence2_model, florence2_processor, image) @@ -185,7 +185,7 @@ def dense_region_caption_and_segmentation( text_input=None, output_dir=OUTPUT_DIR ): - assert text_input is None, "Text input should not be none when calling dense region caption pipeline." + assert text_input is None, "Text input should be None when calling dense region caption pipeline." # run florence-2 object detection in demo image = Image.open(image_path).convert("RGB") results = run_florence2(task_prompt, text_input, florence2_model, florence2_processor, image) @@ -262,7 +262,7 @@ def region_proposal_and_segmentation( text_input=None, output_dir=OUTPUT_DIR ): - assert text_input is None, "Text input should not be none when calling region proposal pipeline." + assert text_input is None, "Text input should be None when calling region proposal pipeline." # run florence-2 object detection in demo image = Image.open(image_path).convert("RGB") results = run_florence2(task_prompt, text_input, florence2_model, florence2_processor, image) @@ -355,7 +355,7 @@ def phrase_grounding_and_segmentation( } } """ - assert text_input is not None, "Text input should not be none when calling phrase grounding pipeline." + assert text_input is not None, "Text input should not be None when calling phrase grounding pipeline." results = results[task_prompt] # parse florence-2 detection results input_boxes = np.array(results["bboxes"]) @@ -428,7 +428,7 @@ def referring_expression_segmentation( } } """ - assert text_input is not None, "Text input should not be none when calling referring segmentation pipeline." + assert text_input is not None, "Text input should not be None when calling referring segmentation pipeline." results = results[task_prompt] # parse florence-2 detection results polygon_points = np.array(results["polygons"][0], dtype=np.int32).reshape(-1, 2) @@ -542,7 +542,7 @@ def open_vocabulary_detection_and_segmentation( } } """ - assert text_input is not None, "Text input should not be none when calling open-vocabulary detection pipeline." + assert text_input is not None, "Text input should not be None when calling open-vocabulary detection pipeline." results = results[task_prompt] # parse florence-2 detection results input_boxes = np.array(results["bboxes"]) From 4f3adf3222ac777fadef9655d82818b2773bd2db Mon Sep 17 00:00:00 2001 From: rentainhe <596106517@qq.com> Date: Sat, 31 Aug 2024 20:22:17 +0800 Subject: [PATCH 2/5] support dump results in 1.5 image demo --- grounded_sam2_gd1.5_demo.py | 70 ++++++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 8 deletions(-) diff --git a/grounded_sam2_gd1.5_demo.py b/grounded_sam2_gd1.5_demo.py index 95bbf1b..e699498 100644 --- a/grounded_sam2_gd1.5_demo.py +++ b/grounded_sam2_gd1.5_demo.py @@ -6,19 +6,38 @@ from dds_cloudapi_sdk import TextPrompt from dds_cloudapi_sdk import DetectionModel from dds_cloudapi_sdk import DetectionTarget +import os import cv2 +import json import torch import numpy as np import supervision as sv +import pycocotools.mask as mask_util +from pathlib import Path from PIL import Image from sam2.build_sam import build_sam2 from sam2.sam2_image_predictor import SAM2ImagePredictor +""" +Hyper parameters +""" +API_TOKEN = "Your API token" +TEXT_PROMPT = "car" +IMG_PATH = "notebooks/images/cars.jpg" +SAM2_CHECKPOINT = "./checkpoints/sam2_hiera_large.pt" +SAM2_MODEL_CONFIG = "sam2_hiera_l.yaml" +GROUNDING_MODEL = DetectionModel.GDino1_5_Pro # DetectionModel.GDino1_6_Pro +OUTPUT_DIR = Path("outputs/grounded_sam2_gd1.5_demo") +DUMP_JSON_RESULTS = True + +# create output directory +OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + """ Prompt Grounding DINO 1.5 with Text for Box Prompt Generation with Cloud API """ # Step 1: initialize the config -token = "Your API token" +token = API_TOKEN config = Config(token) # Step 2: initialize the client @@ -27,14 +46,14 @@ client = Client(config) # Step 3: run the task by DetectionTask class # image_url = "https://algosplt.oss-cn-shenzhen.aliyuncs.com/test_files/tasks/detection/iron_man.jpg" # if you are processing local image file, upload them to DDS server to get the image url -img_path = "notebooks/images/cars.jpg" +img_path = IMG_PATH image_url = client.upload_file(img_path) task = DetectionTask( image_url=image_url, - prompts=[TextPrompt(text="car")], + prompts=[TextPrompt(text=TEXT_PROMPT)], targets=[DetectionTarget.BBox], # detect bbox - model=DetectionModel.GDino1_5_Pro, # detect with GroundingDino-1.5-Pro model + model=GROUNDING_MODEL, # detect with GroundingDino-1.5-Pro model ) client.run_task(task) @@ -68,8 +87,8 @@ if torch.cuda.get_device_properties(0).major >= 8: torch.backends.cudnn.allow_tf32 = True # build SAM2 image predictor -sam2_checkpoint = "./checkpoints/sam2_hiera_large.pt" -model_cfg = "sam2_hiera_l.yaml" +sam2_checkpoint = SAM2_CHECKPOINT +model_cfg = SAM2_MODEL_CONFIG sam2_model = build_sam2(model_cfg, sam2_checkpoint, device="cuda") sam2_predictor = SAM2ImagePredictor(sam2_model) @@ -120,8 +139,43 @@ annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections label_annotator = sv.LabelAnnotator() annotated_frame = label_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels) -cv2.imwrite("groundingdino_annotated_image.jpg", annotated_frame) +cv2.imwrite(os.path.join(OUTPUT_DIR, "groundingdino_annotated_image.jpg"), annotated_frame) mask_annotator = sv.MaskAnnotator() annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections) -cv2.imwrite("grounded_sam2_annotated_image_with_mask.jpg", annotated_frame) +cv2.imwrite(os.path.join(OUTPUT_DIR, "grounded_sam2_annotated_image_with_mask.jpg"), annotated_frame) + +""" +Dump the results in standard format and save as json files +""" + +def single_mask_to_rle(mask): + rle = mask_util.encode(np.array(mask[:, :, None], order="F", dtype="uint8"))[0] + rle["counts"] = rle["counts"].decode("utf-8") + return rle + +if DUMP_JSON_RESULTS: + # convert mask into rle format + mask_rles = [single_mask_to_rle(mask) for mask in masks] + + input_boxes = input_boxes.tolist() + scores = scores.tolist() + # save the results in standard format + results = { + "image_path": img_path, + "annotations" : [ + { + "class_name": class_name, + "bbox": box, + "segmentation": mask_rle, + "score": score, + } + for class_name, box, mask_rle, score in zip(class_names, input_boxes, mask_rles, scores) + ], + "box_format": "xyxy", + "img_width": image.width, + "img_height": image.height, + } + + with open(os.path.join(OUTPUT_DIR, "grounded_sam2_gd1.5_image_demo_results.json"), "w") as f: + json.dump(results, f, indent=4) From a99354bb254c5170183a5c19ff68df40e998dbf2 Mon Sep 17 00:00:00 2001 From: rentainhe <596106517@qq.com> Date: Sat, 31 Aug 2024 20:40:59 +0800 Subject: [PATCH 3/5] add dump results to hf model demo --- grounded_sam2_gd1.5_demo.py | 9 ++-- grounded_sam2_hf_model_demo.py | 78 ++++++++++++++++++++++++++++------ 2 files changed, 72 insertions(+), 15 deletions(-) diff --git a/grounded_sam2_gd1.5_demo.py b/grounded_sam2_gd1.5_demo.py index e699498..a70a49e 100644 --- a/grounded_sam2_gd1.5_demo.py +++ b/grounded_sam2_gd1.5_demo.py @@ -22,11 +22,12 @@ from sam2.sam2_image_predictor import SAM2ImagePredictor Hyper parameters """ API_TOKEN = "Your API token" -TEXT_PROMPT = "car" +TEXT_PROMPT = "car . building ." IMG_PATH = "notebooks/images/cars.jpg" SAM2_CHECKPOINT = "./checkpoints/sam2_hiera_large.pt" SAM2_MODEL_CONFIG = "sam2_hiera_l.yaml" GROUNDING_MODEL = DetectionModel.GDino1_5_Pro # DetectionModel.GDino1_6_Pro +DEVICE = "cuda" if torch.cuda.is_available() else "cpu" OUTPUT_DIR = Path("outputs/grounded_sam2_gd1.5_demo") DUMP_JSON_RESULTS = True @@ -79,7 +80,7 @@ Init SAM 2 Model and Predict Mask with Box Prompt # environment settings # use bfloat16 -torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__() +torch.autocast(device_type=DEVICE, dtype=torch.bfloat16).__enter__() if torch.cuda.get_device_properties(0).major >= 8: # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices) @@ -89,7 +90,7 @@ if torch.cuda.get_device_properties(0).major >= 8: # build SAM2 image predictor sam2_checkpoint = SAM2_CHECKPOINT model_cfg = SAM2_MODEL_CONFIG -sam2_model = build_sam2(model_cfg, sam2_checkpoint, device="cuda") +sam2_model = build_sam2(model_cfg, sam2_checkpoint, device=DEVICE) sam2_predictor = SAM2ImagePredictor(sam2_model) image = Image.open(img_path) @@ -160,6 +161,8 @@ if DUMP_JSON_RESULTS: input_boxes = input_boxes.tolist() scores = scores.tolist() + # FIXME: class_names should be a list of strings without spaces + class_names = [class_name.strip() for class_name in class_names] # save the results in standard format results = { "image_path": img_path, diff --git a/grounded_sam2_hf_model_demo.py b/grounded_sam2_hf_model_demo.py index 273a6e6..2bca9db 100644 --- a/grounded_sam2_hf_model_demo.py +++ b/grounded_sam2_hf_model_demo.py @@ -1,7 +1,11 @@ +import os import cv2 +import json import torch import numpy as np import supervision as sv +import pycocotools.mask as mask_util +from pathlib import Path from supervision.draw.color import ColorPalette from utils.supervision_utils import CUSTOM_COLOR_MAP from PIL import Image @@ -9,9 +13,24 @@ from sam2.build_sam import build_sam2 from sam2.sam2_image_predictor import SAM2ImagePredictor from transformers import AutoProcessor, AutoModelForZeroShotObjectDetection +""" +Hyper parameters +""" +GROUNDING_MODEL = "IDEA-Research/grounding-dino-tiny" +TEXT_PROMPT = "car. tire." +IMG_PATH = "notebooks/images/truck.jpg" +SAM2_CHECKPOINT = "./checkpoints/sam2_hiera_large.pt" +SAM2_MODEL_CONFIG = "sam2_hiera_l.yaml" +DEVICE = "cuda" if torch.cuda.is_available() else "cpu" +OUTPUT_DIR = Path("outputs/grounded_sam2_hf_model_demo") +DUMP_JSON_RESULTS = True + +# create output directory +OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + # environment settings # use bfloat16 -torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__() +torch.autocast(device_type=DEVICE, dtype=torch.bfloat16).__enter__() if torch.cuda.get_device_properties(0).major >= 8: # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices) @@ -19,28 +38,27 @@ if torch.cuda.get_device_properties(0).major >= 8: torch.backends.cudnn.allow_tf32 = True # build SAM2 image predictor -sam2_checkpoint = "./checkpoints/sam2_hiera_large.pt" -model_cfg = "sam2_hiera_l.yaml" -sam2_model = build_sam2(model_cfg, sam2_checkpoint, device="cuda") +sam2_checkpoint = SAM2_CHECKPOINT +model_cfg = SAM2_MODEL_CONFIG +sam2_model = build_sam2(model_cfg, sam2_checkpoint, device=DEVICE) sam2_predictor = SAM2ImagePredictor(sam2_model) # build grounding dino from huggingface -model_id = "IDEA-Research/grounding-dino-tiny" -device = "cuda" if torch.cuda.is_available() else "cpu" +model_id = GROUNDING_MODEL processor = AutoProcessor.from_pretrained(model_id) -grounding_model = AutoModelForZeroShotObjectDetection.from_pretrained(model_id).to(device) +grounding_model = AutoModelForZeroShotObjectDetection.from_pretrained(model_id).to(DEVICE) # setup the input image and text prompt for SAM 2 and Grounding DINO # VERY important: text queries need to be lowercased + end with a dot -text = "car. tire." -img_path = 'notebooks/images/truck.jpg' +text = TEXT_PROMPT +img_path = IMG_PATH image = Image.open(img_path) sam2_predictor.set_image(np.array(image.convert("RGB"))) -inputs = processor(images=image, text=text, return_tensors="pt").to(device) +inputs = processor(images=image, text=text, return_tensors="pt").to(DEVICE) with torch.no_grad(): outputs = grounding_model(**inputs) @@ -114,8 +132,44 @@ annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections label_annotator = sv.LabelAnnotator(color=ColorPalette.from_hex(CUSTOM_COLOR_MAP)) annotated_frame = label_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels) -cv2.imwrite("groundingdino_annotated_image.jpg", annotated_frame) +cv2.imwrite(os.path.join(OUTPUT_DIR, "groundingdino_annotated_image.jpg"), annotated_frame) mask_annotator = sv.MaskAnnotator(color=ColorPalette.from_hex(CUSTOM_COLOR_MAP)) annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections) -cv2.imwrite("grounded_sam2_annotated_image_with_mask.jpg", annotated_frame) +cv2.imwrite(os.path.join(OUTPUT_DIR, "grounded_sam2_annotated_image_with_mask.jpg"), annotated_frame) + + +""" +Dump the results in standard format and save as json files +""" + +def single_mask_to_rle(mask): + rle = mask_util.encode(np.array(mask[:, :, None], order="F", dtype="uint8"))[0] + rle["counts"] = rle["counts"].decode("utf-8") + return rle + +if DUMP_JSON_RESULTS: + # convert mask into rle format + mask_rles = [single_mask_to_rle(mask) for mask in masks] + + input_boxes = input_boxes.tolist() + scores = scores.tolist() + # save the results in standard format + results = { + "image_path": img_path, + "annotations" : [ + { + "class_name": class_name, + "bbox": box, + "segmentation": mask_rle, + "score": score, + } + for class_name, box, mask_rle, score in zip(class_names, input_boxes, mask_rles, scores) + ], + "box_format": "xyxy", + "img_width": image.width, + "img_height": image.height, + } + + with open(os.path.join(OUTPUT_DIR, "grounded_sam2_hf_model_demo_results.json"), "w") as f: + json.dump(results, f, indent=4) From e0daab208a87aab82195e3399dafb6372e36a6af Mon Sep 17 00:00:00 2001 From: rentainhe <596106517@qq.com> Date: Sat, 31 Aug 2024 20:55:49 +0800 Subject: [PATCH 4/5] support dump results in local demo --- grounded_sam2_local_demo.py | 83 ++++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 14 deletions(-) diff --git a/grounded_sam2_local_demo.py b/grounded_sam2_local_demo.py index 2735e85..0087d61 100644 --- a/grounded_sam2_local_demo.py +++ b/grounded_sam2_local_demo.py @@ -1,35 +1,55 @@ +import os import cv2 +import json import torch import numpy as np import supervision as sv +import pycocotools.mask as mask_util +from pathlib import Path from torchvision.ops import box_convert from sam2.build_sam import build_sam2 from sam2.sam2_image_predictor import SAM2ImagePredictor from grounding_dino.groundingdino.util.inference import load_model, load_image, predict +""" +Hyper parameters +""" +TEXT_PROMPT = "car. tire." +IMG_PATH = "notebooks/images/truck.jpg" +SAM2_CHECKPOINT = "./checkpoints/sam2_hiera_large.pt" +SAM2_MODEL_CONFIG = "sam2_hiera_l.yaml" +GROUNDING_DINO_CONFIG = "grounding_dino/groundingdino/config/GroundingDINO_SwinT_OGC.py" +GROUNDING_DINO_CHECKPOINT = "gdino_checkpoints/groundingdino_swint_ogc.pth" +BOX_THRESHOLD = 0.35 +TEXT_THRESHOLD = 0.25 +DEVICE = "cuda" if torch.cuda.is_available() else "cpu" +OUTPUT_DIR = Path("outputs/grounded_sam2_local_demo") +DUMP_JSON_RESULTS = True + +# create output directory +OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + # environment settings # use bfloat16 # build SAM2 image predictor -sam2_checkpoint = "./checkpoints/sam2_hiera_large.pt" -model_cfg = "sam2_hiera_l.yaml" -sam2_model = build_sam2(model_cfg, sam2_checkpoint, device="cuda") +sam2_checkpoint = SAM2_CHECKPOINT +model_cfg = SAM2_MODEL_CONFIG +sam2_model = build_sam2(model_cfg, sam2_checkpoint, device=DEVICE) sam2_predictor = SAM2ImagePredictor(sam2_model) # build grounding dino model -model_id = "IDEA-Research/grounding-dino-tiny" -device = "cuda" if torch.cuda.is_available() else "cpu" grounding_model = load_model( - model_config_path="grounding_dino/groundingdino/config/GroundingDINO_SwinT_OGC.py", - model_checkpoint_path="gdino_checkpoints/groundingdino_swint_ogc.pth", - device=device + model_config_path=GROUNDING_DINO_CONFIG, + model_checkpoint_path=GROUNDING_DINO_CHECKPOINT, + device=DEVICE ) # setup the input image and text prompt for SAM 2 and Grounding DINO # VERY important: text queries need to be lowercased + end with a dot -text = "car. tire." -img_path = 'notebooks/images/truck.jpg' +text = TEXT_PROMPT +img_path = IMG_PATH image_source, image = load_image(img_path) @@ -39,8 +59,8 @@ boxes, confidences, labels = predict( model=grounding_model, image=image, caption=text, - box_threshold=0.35, - text_threshold=0.25 + box_threshold=BOX_THRESHOLD, + text_threshold=TEXT_THRESHOLD, ) # process the box prompt for SAM 2 @@ -98,8 +118,43 @@ annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections label_annotator = sv.LabelAnnotator() annotated_frame = label_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels) -cv2.imwrite("groundingdino_annotated_image.jpg", annotated_frame) +cv2.imwrite(os.path.join(OUTPUT_DIR, "groundingdino_annotated_image.jpg"), annotated_frame) mask_annotator = sv.MaskAnnotator() annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections) -cv2.imwrite("grounded_sam2_annotated_image_with_mask.jpg", annotated_frame) +cv2.imwrite(os.path.join(OUTPUT_DIR, "grounded_sam2_annotated_image_with_mask.jpg"), annotated_frame) + +""" +Dump the results in standard format and save as json files +""" + +def single_mask_to_rle(mask): + rle = mask_util.encode(np.array(mask[:, :, None], order="F", dtype="uint8"))[0] + rle["counts"] = rle["counts"].decode("utf-8") + return rle + +if DUMP_JSON_RESULTS: + # convert mask into rle format + mask_rles = [single_mask_to_rle(mask) for mask in masks] + + input_boxes = input_boxes.tolist() + scores = scores.tolist() + # save the results in standard format + results = { + "image_path": img_path, + "annotations" : [ + { + "class_name": class_name, + "bbox": box, + "segmentation": mask_rle, + "score": score, + } + for class_name, box, mask_rle, score in zip(class_names, input_boxes, mask_rles, scores) + ], + "box_format": "xyxy", + "img_width": w, + "img_height": h, + } + + with open(os.path.join(OUTPUT_DIR, "grounded_sam2_local_image_demo_results.json"), "w") as f: + json.dump(results, f, indent=4) \ No newline at end of file From daf5bb3f9799c75c66002f04eb7b1950ecdb0f27 Mon Sep 17 00:00:00 2001 From: rentainhe <596106517@qq.com> Date: Sat, 31 Aug 2024 20:58:37 +0800 Subject: [PATCH 5/5] support dump results in local demo --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0b94a64..276470f 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Grounded SAM 2 does not introduce significant methodological changes compared to ## News +- `2024/08/31`: Support `dump json results` in Grounded SAM 2 Image Demos (with Grounding DINO). - `2024/08/20`: Support **Florence-2 SAM 2 Image Demo** which includes `dense region caption`, `object detection`, `phrase grounding`, and cascaded auto-label pipeline `caption + phrase grounding`. - `2024/08/09`: Support **Ground and Track New Object** throughout the whole videos. This feature is still under development now. Credits to [Shuo Shen](https://github.com/ShuoShenDe). - `2024/08/07`: Support **Custom Video Inputs**, users need only submit their video file (e.g. `.mp4` file) with specific text prompts to get an impressive demo videos.