From b2b81980f86a3ab7ae439b299a2b422cbd489b8f Mon Sep 17 00:00:00 2001 From: Shuo Shen Date: Fri, 9 Aug 2024 02:54:02 +0200 Subject: [PATCH 1/3] fix:update README --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6fcc01e..edb27ea 100644 --- a/README.md +++ b/README.md @@ -186,8 +186,11 @@ You can customize various parameters including: - `text_threshold`: text threshold for groundingdino model Note: This method supports only the mask type of text prompt. +The demo video is: +[![car tracking demo data](./assets/tracking_car_1.jpg)](./assets/tracking_car.mp4) + + After running our demo code, you can get the tracking results as follows: -[![car tracking demo data](./assets/tracking_video_1.jpg)](./assets/tracking_video.mp4) [![car tracking result data](./assets/tracking_car_mask_1.jpg)](./assets/tracking_car_output.mp4) From 0ee1d395098eacd40988374cc1422a037bdb2f94 Mon Sep 17 00:00:00 2001 From: SusanSHEN Date: Fri, 9 Aug 2024 22:23:01 +0200 Subject: [PATCH 2/3] fix:add makefile and dockerfile --- Dockerfile | 34 +++++++++++++++++++++ Makefile | 47 +++++++++++++++++++++++++++++ gdino_checkpoints/download_ckpts.sh | 24 +++++++++++++++ grounding_dino/requirements.txt | 2 +- grounding_dino/setup.py | 6 +++- setup.py | 21 +++++++++++-- 6 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 gdino_checkpoints/download_ckpts.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e04786b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +FROM pytorch/pytorch:2.3.1-cuda12.1-cudnn8-devel + +# Arguments to build Docker Image using CUDA +ARG USE_CUDA=0 +ARG TORCH_ARCH="7.0;7.5;8.0;8.6" + +ENV AM_I_DOCKER True +ENV BUILD_WITH_CUDA "${USE_CUDA}" +ENV TORCH_CUDA_ARCH_LIST="${TORCH_ARCH}" +ENV CUDA_HOME /usr/local/cuda-12.1/ +# Ensure CUDA is correctly set up +ENV PATH /usr/local/cuda-12.1/bin:${PATH} +ENV LD_LIBRARY_PATH /usr/local/cuda-12.1/lib64:${LD_LIBRARY_PATH} + +# Install required packages and specific gcc/g++ +RUN apt-get update && apt-get install --no-install-recommends wget ffmpeg=7:* \ + libsm6=2:* libxext6=2:* git=1:* nano vim=2:* ninja-build gcc-10 g++-10 -y \ + && apt-get clean && apt-get autoremove && rm -rf /var/lib/apt/lists/* + +ENV CC=gcc-10 +ENV CXX=g++-10 + +RUN mkdir -p /home/appuser/Grounded-SAM-2 +COPY . /home/appuser/Grounded-SAM-2/ + +WORKDIR /home/appuser/Grounded-SAM-2 + + +# Install essential Python packages +RUN python -m pip install --upgrade pip setuptools wheel numpy + +# Install segment_anything package in editable mode +RUN python -m pip install -e . + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..15157f1 --- /dev/null +++ b/Makefile @@ -0,0 +1,47 @@ +# Get version of CUDA and enable it for compilation if CUDA > 11.0 +# This solves https://github.com/IDEA-Research/Grounded-Segment-Anything/issues/53 +# and https://github.com/IDEA-Research/Grounded-Segment-Anything/issues/84 +# when running in Docker +# Check if nvcc is installed +NVCC := $(shell which nvcc) +ifeq ($(NVCC),) + # NVCC not found + USE_CUDA := 0 + NVCC_VERSION := "not installed" +else + NVCC_VERSION := $(shell nvcc --version | grep -oP 'release \K[0-9.]+') + USE_CUDA := $(shell echo "$(NVCC_VERSION) > 11" | bc -l) +endif + +# Add the list of supported ARCHs +ifeq ($(USE_CUDA), 1) + TORCH_CUDA_ARCH_LIST := "7.0;7.5;8.0;8.6+PTX" + BUILD_MESSAGE := "I will try to build the image with CUDA support" +else + TORCH_CUDA_ARCH_LIST := + BUILD_MESSAGE := "CUDA $(NVCC_VERSION) is not supported" +endif + + +prepare-checkpoints: + @echo "Preparing checkpoints..." + cd checkpoints && \ + bash download_ckpts.sh && \ + mkdir -p gdino_checkpoints && \ + cd gdino_checkpoints && \ + wget https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth && \ + wget https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha2/groundingdino_swinb_cogcoor.pth + + +build-image: + @echo $(BUILD_MESSAGE) + docker build --build-arg USE_CUDA=$(USE_CUDA) \ + --build-arg TORCH_ARCH=$(TORCH_CUDA_ARCH_LIST) \ + -t grounded_sam2:1.0 . +run: + docker run --gpus all -it --rm --net=host --privileged \ + -v /tmp/.X11-unix:/tmp/.X11-unix \ + -v "${PWD}":/home/appuser/Grounded-SAM-2 \ + -e DISPLAY=$DISPLAY \ + --name=gsa \ + --ipc=host -it grounded_sam2:1.0 diff --git a/gdino_checkpoints/download_ckpts.sh b/gdino_checkpoints/download_ckpts.sh new file mode 100644 index 0000000..2c6c795 --- /dev/null +++ b/gdino_checkpoints/download_ckpts.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + + +# Define the URLs for the checkpoints +BASE_URL="https://github.com/IDEA-Research/GroundingDINO/releases/download/" +swint_ogc_url="${BASE_URL}v0.1.0-alpha/groundingdino_swint_ogc.pth" +swinb_cogcoor_url="${BASE_URL}v0.1.0-alpha2/groundingdino_swinb_cogcoor.pth" + + + +# Download each of the four checkpoints using wget +echo "Downloading groundingdino_swint_ogc.pth checkpoint..." +wget $swint_ogc_url || { echo "Failed to download checkpoint from $swint_ogc_url"; exit 1; } + +echo "Downloading groundingdino_swinb_cogcoor.pth checkpoint..." +wget $swinb_cogcoor_url || { echo "Failed to download checkpoint from $swinb_cogcoor_url"; exit 1; } + +echo "All checkpoints are downloaded successfully." diff --git a/grounding_dino/requirements.txt b/grounding_dino/requirements.txt index 24aa11d..c4915fe 100644 --- a/grounding_dino/requirements.txt +++ b/grounding_dino/requirements.txt @@ -7,4 +7,4 @@ timm numpy opencv-python supervision>=0.22.0 -pycocotools +pycocotools \ No newline at end of file diff --git a/grounding_dino/setup.py b/grounding_dino/setup.py index 275b6fc..5acf018 100644 --- a/grounding_dino/setup.py +++ b/grounding_dino/setup.py @@ -92,6 +92,10 @@ def get_extensions(): "-D__CUDA_NO_HALF_OPERATORS__", "-D__CUDA_NO_HALF_CONVERSIONS__", "-D__CUDA_NO_HALF2_OPERATORS__", + "-gencode=arch=compute_70,code=sm_70", + "-gencode=arch=compute_75,code=sm_75", + "-gencode=arch=compute_80,code=sm_80", + "-gencode=arch=compute_86,code=sm_86", ] else: print("Compiling without CUDA") @@ -208,7 +212,7 @@ if __name__ == "__main__": url="https://github.com/IDEA-Research/GroundingDINO", description="open-set object detector", license=license, - install_requires=parse_requirements("requirements.txt"), + # install_requires=parse_requirements("requirements.txt"), packages=find_packages( exclude=( "configs", diff --git a/setup.py b/setup.py index 94f41b5..d49ddc5 100644 --- a/setup.py +++ b/setup.py @@ -23,13 +23,30 @@ with open("README.md", "r") as f: # Required dependencies REQUIRED_PACKAGES = [ - "torch>=2.3.1", + "torch>=2.3.1", "torchvision>=0.18.1", + "transformers", "numpy>=1.24.4", "tqdm>=4.66.1", "hydra-core>=1.3.2", "iopath>=0.1.10", - "pillow>=9.4.0", + "pillow>=9.4.0", + "huggingface_hub", + "diffusers[torch]==0.15.1", + "onnxruntime==1.14.1", + "onnx==1.13.1", + "ipykernel==6.16.2", + "scipy", + "gradio", + "openai", + "matplotlib>=3.9.1", + "opencv-python>=4.7.0", + "dds_cloudapi_sdk", + "addict", + "yapf", + "timm", + "supervision>=0.22.0", + "pycocotools", ] EXTRA_PACKAGES = { From 0f48470ec31ce0fe44920b813fdbb3be6acc9620 Mon Sep 17 00:00:00 2001 From: SusanSHEN Date: Fri, 9 Aug 2024 22:39:52 +0200 Subject: [PATCH 3/3] update README --- Makefile | 10 ---------- README.md | 36 ++++++++++++++++++++++++++---------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 15157f1..7aab9ed 100644 --- a/Makefile +++ b/Makefile @@ -23,16 +23,6 @@ else endif -prepare-checkpoints: - @echo "Preparing checkpoints..." - cd checkpoints && \ - bash download_ckpts.sh && \ - mkdir -p gdino_checkpoints && \ - cd gdino_checkpoints && \ - wget https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth && \ - wget https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha2/groundingdino_swinb_cogcoor.pth - - build-image: @echo $(BUILD_MESSAGE) docker build --build-arg USE_CUDA=$(USE_CUDA) \ diff --git a/README.md b/README.md index da18ea0..3fd24d3 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,22 @@ Grounded SAM 2 does not introduce significant methodological changes compared to ## Installation +Download the pretrained `SAM 2` checkpoints: + +```bash +cd checkpoints +bash download_ckpts.sh +``` + +Download the pretrained `Grounding DINO` checkpoints: + +```bash +cd gdino_checkpoints +bash download_ckpts.sh +``` + +### Installation without docker + Install PyTorch environment first. We use `python=3.10`, as well as `torch >= 2.3.1`, `torchvision>=0.18.1` and `cuda-12.1` in our environment to run this demo. Please follow the instructions [here](https://pytorch.org/get-started/locally/) to install both PyTorch and TorchVision dependencies. Installing both PyTorch and TorchVision with CUDA support is strongly recommended. You can easily install the latest version of PyTorch as follows: ```bash @@ -56,19 +72,19 @@ Install `Grounding DINO`: pip install --no-build-isolation -e grounding_dino ``` -Download the pretrained `SAM 2` checkpoints: +### Installation with docker +Build the Docker image and Run the Docker container: -```bash -cd checkpoints -bash download_ckpts.sh ``` +cd Grounded-SAM-2 +make build-image +make run +``` +After executing these commands, you will be inside the Docker environment. The working directory within the container is set to: `/home/appuser/Grounded-SAM-2` -Download the pretrained `Grounding DINO` checkpoints: - -```bash -cd gdino_checkpoints -wget https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth -wget https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha2/groundingdino_swinb_cogcoor.pth +Once inside the Docker environment, you can start the demo by running: +``` +python grounded_sam2_tracking_demo.py ``` ## Grounded SAM 2 Demos