add grounded samurai demo with dino-x

Update the demo.py

.gitignore

@@ -1,15 +1,3 @@
-# SAM 2
-.vscode/
-.DS_Store
-__pycache__/
-*-checkpoint.ipynb
-.venv
-*.egg*
-build/*
-_C.*
-outputs/*
-checkpoints/*.pt
-*test*
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -26,13 +14,11 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
 lib64/
 parts/
 sdist/
 var/
 wheels/
-pip-wheel-metadata/
 share/python-wheels/
 *.egg-info/
 .installed.cfg
@@ -62,6 +48,7 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+cover/
 
 # Translations
 *.mo
@@ -84,6 +71,7 @@ instance/
 docs/_build/
 
 # PyBuilder
+.pybuilder/
 target/
 
 # Jupyter Notebook
@@ -94,7 +82,9 @@ profile_default/
 ipython_config.py
 
 # pyenv
-.python-version
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
 
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
@@ -103,7 +93,24 @@ ipython_config.py
 #   install all needed dependencies.
 #Pipfile.lock
 
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
 __pypackages__/
 
 # Celery stuff
@@ -140,8 +147,20 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-# checkpoint
-*.pth
-outputs/
+# pytype static type analyzer
+.pytype/
 
-.idea/
+# Cython debug symbols
+cython_debug/
+
+# evaluation results
+evaluation_results/*
+results/*
+debug/*
+visualization/*
+
+# .DS_Store
+.DS_Store
+
+# For Testing
+demo/

Dockerfile

@@ -1,37 +0,0 @@
-FROM pytorch/pytorch:2.3.1-cuda12.1-cudnn8-devel
-
-# Arguments to build Docker Image using CUDA
-ARG USE_CUDA=0
-ARG TORCH_ARCH="7.0;7.5;8.0;8.6"
-
-ENV AM_I_DOCKER=True
-ENV BUILD_WITH_CUDA="${USE_CUDA}"
-ENV TORCH_CUDA_ARCH_LIST="${TORCH_ARCH}"
-ENV CUDA_HOME=/usr/local/cuda-12.1/
-# Ensure CUDA is correctly set up
-ENV PATH=/usr/local/cuda-12.1/bin:${PATH}
-ENV LD_LIBRARY_PATH=/usr/local/cuda-12.1/lib64:${LD_LIBRARY_PATH}
-
-# Install required packages and specific gcc/g++
-RUN apt-get update && apt-get install --no-install-recommends wget ffmpeg=7:* \
-    libsm6=2:* libxext6=2:* git=1:* nano vim=2:* ninja-build gcc-10 g++-10 -y \
-    && apt-get clean && apt-get autoremove && rm -rf /var/lib/apt/lists/*
-
-ENV CC=gcc-10
-ENV CXX=g++-10
-
-RUN mkdir -p /home/appuser/Grounded-SAM-2
-COPY . /home/appuser/Grounded-SAM-2/
-
-WORKDIR /home/appuser/Grounded-SAM-2
-
-
-# Install essential Python packages
-RUN python -m pip install --upgrade pip setuptools wheel numpy \
-    opencv-python transformers supervision pycocotools addict yapf timm
-
-# Install segment_anything package in editable mode
-RUN python -m pip install -e .
-
-# Install grounding dino 
-RUN python -m pip install --no-build-isolation -e grounding_dino

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2023 - present, IDEA Research.
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -198,4 +198,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.

LICENSE_groundingdino

@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright 2023 - present, IDEA Research.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

Makefile

@@ -1,37 +0,0 @@
-# Get version of CUDA and enable it for compilation if CUDA > 11.0
-# This solves https://github.com/IDEA-Research/Grounded-Segment-Anything/issues/53
-# and https://github.com/IDEA-Research/Grounded-Segment-Anything/issues/84
-# when running in Docker
-# Check if nvcc is installed
-NVCC := $(shell which nvcc)
-ifeq ($(NVCC),)
-	# NVCC not found
-	USE_CUDA := 0
-	NVCC_VERSION := "not installed"
-else
-	NVCC_VERSION := $(shell nvcc --version | grep -oP 'release \K[0-9.]+')
-	USE_CUDA := $(shell echo "$(NVCC_VERSION) > 11" | bc -l)
-endif
-
-# Add the list of supported ARCHs
-ifeq ($(USE_CUDA), 1)
-	TORCH_CUDA_ARCH_LIST := "7.0;7.5;8.0;8.6+PTX"
-	BUILD_MESSAGE := "I will try to build the image with CUDA support"
-else
-	TORCH_CUDA_ARCH_LIST :=
-	BUILD_MESSAGE := "CUDA $(NVCC_VERSION) is not supported"
-endif
-
-
-build-image:
-	@echo $(BUILD_MESSAGE)
-	docker build --build-arg USE_CUDA=$(USE_CUDA) \
-	--build-arg TORCH_ARCH=$(TORCH_CUDA_ARCH_LIST) \
-	-t grounded_sam2:1.0 .
-run:
-	docker run --gpus all -it --rm --net=host --privileged \
-	-v /tmp/.X11-unix:/tmp/.X11-unix \
-	-v "${PWD}":/home/appuser/Grounded-SAM-2 \
-	-e DISPLAY=$DISPLAY \
-	--name=gsa \
-	--ipc=host -it grounded_sam2:1.0

README.md

@@ -1,484 +1,28 @@
-# Grounded SAM 2: Ground and Track Anything in Videos
-
-**[IDEA-Research](https://github.com/idea-research)**
-
-[Tianhe Ren](https://rentainhe.github.io/), [Shuo Shen](https://github.com/ShuoShenDe)
-
-[[`SAM 2 Paper`](https://arxiv.org/abs/2408.00714)] [[`Grounding DINO Paper`](https://arxiv.org/abs/2303.05499)] [[`Grounding DINO 1.5 Paper`](https://arxiv.org/abs/2405.10300)] [[`DINO-X Paper`](https://arxiv.org/abs/2411.14347)] [[`BibTeX`](#citation)]
-
-[![Video Name](./assets/grounded_sam_2_intro.jpg)](https://github.com/user-attachments/assets/f0fb0022-779a-49fb-8f46-3a18a8b4e893)
-
-## Highlights
-
- Grounded SAM 2 is a foundation model pipeline towards grounding and track anything in Videos with [Grounding DINO](https://arxiv.org/abs/2303.05499), [Grounding DINO 1.5](https://arxiv.org/abs/2405.10300), [Florence-2](https://arxiv.org/abs/2311.06242), [DINO-X](https://arxiv.org/abs/2411.14347) and [SAM 2](https://arxiv.org/abs/2408.00714).
-
-In this repo, we've supported the following demo with **simple implementations**:
-- **Ground and Segment Anything** with Grounding DINO, Grounding DINO 1.5 & 1.6, DINO-X and SAM 2
-- **Ground and Track Anything** with Grounding DINO, Grounding DINO 1.5 & 1.6, DINO-X and SAM 2
-- **Detect, Segment and Track Visualization** based on the powerful [supervision](https://github.com/roboflow/supervision) library.
-
-Grounded SAM 2 does not introduce significant methodological changes compared to [Grounded SAM: Assembling Open-World Models for Diverse Visual Tasks](https://arxiv.org/abs/2401.14159). Both approaches leverage the capabilities of open-world models to address complex visual tasks. Consequently, we try to **simplify the code implementation** in this repository, aiming to enhance user convenience.
-
-## Latest updates
-
-- **2024.12.02**: Support **DINO-X with SAM 2** demos (including object segmentation and tracking), please install the latest version of `dds-cloudapi-sdk==0.3.3` and refer to [Grounded SAM 2 (with DINO-X)](#grounded-sam-2-image-demo-with-dino-x) and [Grounded SAM 2 Video (with DINO-X)](#grounded-sam-2-video-object-tracking-demo-with-custom-video-input-with-dino-x) for more details.
-
-- **2024.10.24**: Support [SAHI (Slicing Aided Hyper Inference)](https://docs.ultralytics.com/guides/sahi-tiled-inference/) on Grounded SAM 2 (with Grounding DINO 1.5) which may be helpful for inferencing high resolution image with dense small objects (e.g. **4K** images).
-
-- **2024.10.10**: Support `SAM-2.1` models, if you want to use `SAM 2.1` model, you need to update to the latest code and reinstall SAM 2 follow [SAM 2.1 Installation](https://github.com/facebookresearch/sam2?tab=readme-ov-file#latest-updates).
-
-- **2024.08.31**: Support `dump json results` in Grounded SAM 2 Image Demos (with Grounding DINO).
-
-- **2024.08.20**: Support **Florence-2 SAM 2 Image Demo** which includes `dense region caption`, `object detection`, `phrase grounding`, and cascaded auto-label pipeline `caption + phrase grounding`.
-
-- **2024.08.09**: Support **Ground and Track New Object** throughout the whole videos. This feature is still under development now. Credits to [Shuo Shen](https://github.com/ShuoShenDe).
-
-- **2024.08.07**: Support **Custom Video Inputs**, users need only submit their video file (e.g. `.mp4` file) with specific text prompts to get an impressive demo videos.
-
-## Contents
-- [Installation](#installation)
-- [Grounded SAM 2 Demos](#grounded-sam-2-demos)
-  - [Grounded SAM 2 Image Demo](#grounded-sam-2-image-demo-with-grounding-dino)
-  - [Grounded SAM 2 Image Demo (with Grounding DINO 1.5 & 1.6)](#grounded-sam-2-image-demo-with-grounding-dino-15--16)
-  - [Grounded SAM 2 Image Demo (with DINO-X)](#grounded-sam-2-image-demo-with-dino-x)
-  - [Grounded SAM 2 with SAHI for High Resolution Image Inference](#sahi-slicing-aided-hyper-inference-with-grounding-dino-15-and-sam-2)
-  - [Automatically Saving Grounding and Segmentation Results](#automatically-saving-grounding-results-image-demo)
-  - [Grounded SAM 2 Video Object Tracking Demo](#grounded-sam-2-video-object-tracking-demo)
-  - [Grounded SAM 2 Video Object Tracking Demo (with Grounding DINO 1.5 & 1.6)](#grounded-sam-2-video-object-tracking-demo-with-grounding-dino-15--16)
-  - [Grounded SAM 2 Video Object Tracking with Custom Video Input (using Grounding DINO)](#grounded-sam-2-video-object-tracking-demo-with-custom-video-input-with-grounding-dino)
-  - [Grounded SAM 2 Video Object Tracking with Custom Video Input (using Grounding DINO 1.5 & 1.6)](#grounded-sam-2-video-object-tracking-demo-with-custom-video-input-with-grounding-dino-15--16)
-  - [Grounded SAM 2 Video Object Tracking Demo (with DINO-X)](#grounded-sam-2-video-object-tracking-demo-with-custom-video-input-with-dino-x)
-  - [Grounded SAM 2 Video Object Tracking with Continues ID (using Grounding DINO)](#grounded-sam-2-video-object-tracking-with-continuous-id-with-grounding-dino)
-- [Grounded SAM 2 Florence-2 Demos](#grounded-sam-2-florence-2-demos)
-  - [Grounded SAM 2 Florence-2 Image Demo](#grounded-sam-2-florence-2-image-demo)
-  - [Grounded SAM 2 Florence-2 Image Auto-Labeling Demo](#grounded-sam-2-florence-2-image-auto-labeling-demo)
-- [Citation](#citation)
+## Grounded SAMURAI
 
+We have tried to implement Grounded SAMURAI for long video object tracking and segmentation.
 
+[![Video Name]()](https://github.com/user-attachments/assets/51db13b6-1083-4c22-af14-c34e09403591)
 
 ## Installation
 
-Download the pretrained `SAM 2` checkpoints:
+### Install SAMURAI
+Please refer to [SAMURAI Install](./SAMURAI_README.md) for more details.
 
-```bash
-cd checkpoints
-bash download_ckpts.sh
-```
+### Register on Offical Website to Get API Token
 
-Download the pretrained `Grounding DINO` checkpoints:
+- **First-Time Application**: If you are interested in our project and wish to try our algorithm, you will need to apply for the corresponding API Token through our [request API token website](https://cloud.deepdataspace.com/apply-token?from=github) for your first attempt.
 
-```bash
-cd gdino_checkpoints
-bash download_ckpts.sh
-```
+- **Request Additional Token Quotas**: If you find our project helpful and need more API token quotas, you can request additional tokens by [filling out this form](https://docs.google.com/forms/d/e/1FAIpQLSfjogAtkgoVyFX9wvCAE15mD7QtHdKdKOrVmcE5GT1xu-03Aw/viewform?usp=sf_link). Our team will review your request and allocate more tokens for your use in one or two days. You can also apply for more tokens by sending us an email.
 
-### Installation without docker
-
-Install PyTorch environment first. We use `python=3.10`, as well as `torch >= 2.3.1`, `torchvision>=0.18.1` and `cuda-12.1` in our environment to run this demo. Please follow the instructions [here](https://pytorch.org/get-started/locally/) to install both PyTorch and TorchVision dependencies. Installing both PyTorch and TorchVision with CUDA support is strongly recommended. You can easily install the latest version of PyTorch as follows:
-
-```bash
-pip3 install torch torchvision torchaudio
-```
-
-Since we need the CUDA compilation environment to compile the `Deformable Attention` operator used in Grounding DINO, we need to check whether the CUDA environment variables have been set correctly (which you can refer to [Grounding DINO Installation](https://github.com/IDEA-Research/GroundingDINO?tab=readme-ov-file#hammer_and_wrench-install) for more details). You can set the environment variable manually as follows if you want to build a local GPU environment for Grounding DINO to run Grounded SAM 2:
-
-```bash
-export CUDA_HOME=/path/to/cuda-12.1/
-```
-
-Install `Segment Anything 2`:
-
-```bash
-pip install -e .
-```
-
-Install `Grounding DINO`:
-
-```bash
-pip install --no-build-isolation -e grounding_dino
-```
-
-### Installation with docker
-Build the Docker image and Run the Docker container:
-
-```
-cd Grounded-SAM-2
-make build-image
-make run
-```
-After executing these commands, you will be inside the Docker environment. The working directory within the container is set to: `/home/appuser/Grounded-SAM-2`
-
-Once inside the Docker environment, you can start the demo by running:
-```
-python grounded_sam2_tracking_demo.py
-```
-
-## Grounded SAM 2 Demos
-### Grounded SAM 2 Image Demo (with Grounding DINO)
-Note that `Grounding DINO` has already been supported in [Huggingface](https://huggingface.co/IDEA-Research/grounding-dino-tiny), so we provide two choices for running `Grounded SAM 2` model:
-- Use huggingface API to inference Grounding DINO (which is simple and clear)
-
-```bash
-python grounded_sam2_hf_model_demo.py
-```
-
-> [!NOTE]
-> 🚨 If you encounter network issues while using the `HuggingFace` model, you can resolve them by setting the appropriate mirror source as `export HF_ENDPOINT=https://hf-mirror.com`
-
-- Load local pretrained Grounding DINO checkpoint and inference with Grounding DINO original API (make sure you've already downloaded the pretrained checkpoint)
-
-```bash
-python grounded_sam2_local_demo.py
-```
-
-
-### Grounded SAM 2 Image Demo (with Grounding DINO 1.5 & 1.6)
-
-We've already released our most capable open-set detection model [Grounding DINO 1.5 & 1.6](https://github.com/IDEA-Research/Grounding-DINO-1.5-API), which can be combined with SAM 2 for stronger open-set detection and segmentation capability. You can apply the API token first and run Grounded SAM 2 with Grounding DINO 1.5 as follows:
-
-Install the latest DDS cloudapi:
+**Note:** If you encounter some errors with API, please install the latest version of `dds-cloudapi-sdk`:
 
 ```bash
 pip install dds-cloudapi-sdk --upgrade
-```
+``` 
 
-Apply your API token from our official website here: [request API token](https://deepdataspace.com/request_api).
+### Demos
 
 ```bash
-python grounded_sam2_gd1.5_demo.py
-```
-
-### SAHI (Slicing Aided Hyper Inference) with Grounding DINO 1.5 and SAM 2
-
-If your images are high resolution with dense objects, directly using Grounding DINO 1.5 for inference on the original image may not be the best choice. We support [SAHI (Slicing Aided Hyper Inference)](https://docs.ultralytics.com/guides/sahi-tiled-inference/), which works by first dividing the original image into smaller overlapping patches. Inference is then performed separately on each patch, and the final detection results are merged. This method is highly effective and accuracy for dense and small objects detection in high resolution images.
-
-You can run SAHI inference by setting the following param in [grounded_sam2_gd1.5_demo.py](./grounded_sam2_gd1.5_demo.py):
-
-```python
-WITH_SLICE_INFERENCE = True
-```
-
-The visualization is shown as follows:
-
-| Text Prompt | Input Image | Grounded SAM 2 | Grounded SAM 2 with SAHI |
-|:----:|:----:|:----:|:----:|
-| `Person` | ![](https://github.com/IDEA-Research/detrex-storage/blob/main/assets/grounded_sam_2/demo_images/dense%20people.png?raw=true) | ![](https://github.com/IDEA-Research/detrex-storage/blob/main/assets/grounded_sam_2/grounding_dino_1.5_slice_inference/grounded_sam2_annotated_image_with_mask.jpg?raw=true) | ![](https://github.com/IDEA-Research/detrex-storage/blob/main/assets/grounded_sam_2/grounding_dino_1.5_slice_inference/grounded_sam2_annotated_image_with_mask_with_slice_inference.jpg?raw=true) |
-
-- **Notes:** We only support SAHI on Grounding DINO 1.5 because it works better with stronger grounding model which may produce less hallucination results.
-
-### Grounded SAM 2 Image Demo (with DINO-X)
-
-We've implemented Grounded SAM 2 with the strongest open-world perception model [DINO-X](https://github.com/IDEA-Research/DINO-X-API) for better open-set detection and segmentation performance. You can apply the API token first and run Grounded SAM 2 with DINO-X as follows:
-
-Install the latest DDS cloudapi:
-
-```bash
-pip install dds-cloudapi-sdk --upgrade
-```
-
-Apply your API token from our official website here: [request API token](https://deepdataspace.com/request_api).
-
-```bash
-python grounded_sam2_dinox_demo.py
-```
-
-### Automatically Saving Grounding Results (Image Demo)
-
-After setting `DUMP_JSON_RESULTS=True` in the following Grounded SAM 2 Image Demos:
-- [grounded_sam2_local_demo.py](./grounded_sam2_local_demo.py)
-- [grounded_sam2_hf_model_demo.py](./grounded_sam2_hf_model_demo.py)
-- [grounded_sam2_gd1.5_demo.py](./grounded_sam2_gd1.5_demo.py)
-- [grounded_sam2_dinox_demo.py](./grounded_sam2_dinox_demo.py)
-
-The `grounding` and `segmentation` results will be automatically saved in the `outputs` dir with the following format:
-
-```python
-{
-    "image_path": "path/to/image.jpg",
-    "annotations": [
-        {
-            "class_name": "class_name",
-            "bbox": [x1, y1, x2, y2],
-            "segmentation": {
-                "size": [h, w],
-                "counts": "rle_encoded_mask"
-            },
-            "score": confidence score
-        }
-    ],
-    "box_format": "xyxy",
-    "img_width": w,
-    "img_height": h
-}
-```
-
-
-
-### Grounded SAM 2 Video Object Tracking Demo
-
-Based on the strong tracking capability of SAM 2, we can combined it with Grounding DINO for open-set object segmentation and tracking. You can run the following scripts to get the tracking results with Grounded SAM 2:
-
-```bash
-python grounded_sam2_tracking_demo.py
-```
-
-- The tracking results of each frame will be saved in `./tracking_results`
-- The video will be save as `children_tracking_demo_video.mp4`
-- You can refine this file with different text prompt and video clips yourself to get more tracking results.
-- We only prompt the first video frame with Grounding DINO here for simple usage.
-
-#### Support Various Prompt Type for Tracking
-
-We've supported different types of prompt for Grounded SAM 2 tracking demo:
-
-- **Point Prompt**: In order to **get a stable segmentation results**, we re-use the SAM 2 image predictor to get the prediction mask from each object based on Grounding DINO box outputs, then we **uniformly sample points from the prediction mask** as point prompts for SAM 2 video predictor
-- **Box Prompt**: We directly use the box outputs from Grounding DINO as box prompts for SAM 2 video predictor
-- **Mask Prompt**: We use the SAM 2 mask prediction results based on Grounding DINO box outputs as mask prompt for SAM 2 video predictor.
-
-![Grounded SAM 2 Tracking Pipeline](./assets/g_sam2_tracking_pipeline_vis_new.png)
-
-
-### Grounded SAM 2 Video Object Tracking Demo (with Grounding DINO 1.5 & 1.6)
-
-We've also support video object tracking demo based on our stronger `Grounding DINO 1.5` model and `SAM 2`, you can try the following demo after applying the API keys for running `Grounding DINO 1.5`:
-
-```bash
-python grounded_sam2_tracking_demo_with_gd1.5.py
-```
-
-### Grounded SAM 2 Video Object Tracking Demo with Custom Video Input (with Grounding DINO)
-
-Users can upload their own video file (e.g. `assets/hippopotamus.mp4`) and specify their custom text prompts for grounding and tracking with Grounding DINO and SAM 2 by using the following scripts:
-
-```bash
-python grounded_sam2_tracking_demo_custom_video_input_gd1.0_hf_model.py
-```
-
-If you are not convenient to use huggingface demo, you can also run tracking demo with local grounding dino model with the following scripts:
-
-```bash
-python grounded_sam2_tracking_demo_custom_video_input_gd1.0_local_model.py
-```
-
-### Grounded SAM 2 Video Object Tracking Demo with Custom Video Input (with Grounding DINO 1.5 & 1.6)
-
-Users can upload their own video file (e.g. `assets/hippopotamus.mp4`) and specify their custom text prompts for grounding and tracking with Grounding DINO 1.5 and SAM 2 by using the following scripts:
-
-```bash
-python grounded_sam2_tracking_demo_custom_video_input_gd1.5.py
-```
-
-You can specify the params in this file:
-
-```python
-VIDEO_PATH = "./assets/hippopotamus.mp4"
-TEXT_PROMPT = "hippopotamus."
-OUTPUT_VIDEO_PATH = "./hippopotamus_tracking_demo.mp4"
-API_TOKEN_FOR_GD1_5 = "Your API token" # api token for G-DINO 1.5
-PROMPT_TYPE_FOR_VIDEO = "mask" # using SAM 2 mask prediction as prompt for video predictor
-```
-
-After running our demo code, you can get the tracking results as follows:
-
-[![Video Name](./assets/hippopotamus_seg.jpg)](https://github.com/user-attachments/assets/1fbdc6f4-3e50-4221-9600-98c397beecdf)
-
-And we will automatically save the tracking visualization results in `OUTPUT_VIDEO_PATH`.
-
-> [!WARNING]
-> We initialize the box prompts on the first frame of the input video. If you want to start from different frame, you can refine `ann_frame_idx` by yourself in our code.
-
-### Grounded SAM 2 Video Object Tracking Demo with Custom Video Input (with DINO-X)
-
-Users can upload their own video file (e.g. `assets/hippopotamus.mp4`) and specify their custom text prompts for grounding and tracking with DINO-X and SAM 2 by using the following scripts:
-
-```bash
-python grounded_sam2_tracking_demo_custom_video_input_dinox.py
-```
-
-### Grounded-SAM-2 Video Object Tracking with Continuous ID (with Grounding DINO)
-
-In above demos, we only prompt Grounded SAM 2 in specific frame, which may not be friendly to find new object during the whole video. In this demo, we try to **find new objects** and assign them with new ID across the whole video, this function is **still under develop**. it's not that stable now.
-
-Users can upload their own video files and specify custom text prompts for grounding and tracking using the Grounding DINO and SAM 2 frameworks. To do this, execute the script:
-
-
-```bash 
-python grounded_sam2_tracking_demo_with_continuous_id.py
-```
-
-You can customize various parameters including:
-
-- `text`: The grounding text prompt.
-- `video_dir`: Directory containing the video files.
-- `output_dir`: Directory to save the processed output.
-- `output_video_path`: Path for the output video.
-- `step`: Frame stepping for processing.
-- `box_threshold`: box threshold for groundingdino model
-- `text_threshold`: text threshold for groundingdino model
-Note: This method supports only the mask type of text prompt.
-
-After running our demo code, you can get the tracking results as follows:
-
-[![Video Name](./assets/tracking_car_mask_1.jpg)](https://github.com/user-attachments/assets/d3f91ad0-3d32-43c4-a0dc-0bed661415f4)
-
-If you want to try `Grounding DINO 1.5` model, you can run the following scripts after setting your API token:
-
-```bash
-python grounded_sam2_tracking_demo_with_continuous_id_gd1.5.py
-```
-
-### Grounded-SAM-2 Video Object Tracking with Continuous ID plus Reverse Tracking(with Grounding DINO)
-This method could simply cover the whole lifetime of the object
-```bash
-python grounded_sam2_tracking_demo_with_continuous_id_plus.py
-
-```
-
-## Grounded SAM 2 Florence-2 Demos
-### Grounded SAM 2 Florence-2 Image Demo
-
-In this section, we will explore how to integrate the feature-rich and robust open-source models [Florence-2](https://arxiv.org/abs/2311.06242) and SAM 2 to develop practical applications.
-
-[Florence-2](https://arxiv.org/abs/2311.06242) is a powerful vision foundation model by Microsoft which supports a series of vision tasks by prompting with special `task_prompt` includes but not limited to:
-
-| Task | Task Prompt | Text Input | Task Introduction |
-|:---:|:---:|:---:|:---:|
-| Object Detection | `<OD>` | &#10008; | Detect main objects with single category name |
-| Dense Region Caption | `<DENSE_REGION_CAPTION>` | &#10008; | Detect main objects with short description |
-| Region Proposal | `<REGION_PROPOSAL>` | &#10008; | Generate proposals without category name |
-| Phrase Grounding | `<CAPTION_TO_PHRASE_GROUNDING>` | &#10004; | Ground main objects in image mentioned in caption |
-| Referring Expression Segmentation | `<REFERRING_EXPRESSION_SEGMENTATION>` | &#10004; | Ground the object which is most related to the text input |
-| Open Vocabulary Detection and Segmentation | `<OPEN_VOCABULARY_DETECTION>` | &#10004; | Ground any object with text input |
-
-
-Integrate `Florence-2` with `SAM-2`, we can build a strong vision pipeline to solve complex vision tasks, you can try the following scripts to run the demo:
-
-> [!NOTE]
-> 🚨 If you encounter network issues while using the `HuggingFace` model, you can resolve them by setting the appropriate mirror source as `export HF_ENDPOINT=https://hf-mirror.com`
-
-**Object Detection and Segmentation**
-```bash
-python grounded_sam2_florence2_image_demo.py \
-    --pipeline object_detection_segmentation \
-    --image_path ./notebooks/images/cars.jpg
-```
-
-**Dense Region Caption and Segmentation**
-```bash
-python grounded_sam2_florence2_image_demo.py \
-    --pipeline dense_region_caption_segmentation \
-    --image_path ./notebooks/images/cars.jpg
-```
-
-**Region Proposal and Segmentation**
-```bash
-python grounded_sam2_florence2_image_demo.py \
-    --pipeline region_proposal_segmentation \
-    --image_path ./notebooks/images/cars.jpg
-```
-
-**Phrase Grounding and Segmentation**
-```bash
-python grounded_sam2_florence2_image_demo.py \
-    --pipeline phrase_grounding_segmentation \
-    --image_path ./notebooks/images/cars.jpg \
-    --text_input "The image shows two vintage Chevrolet cars parked side by side, with one being a red convertible and the other a pink sedan, \
-            set against the backdrop of an urban area with a multi-story building and trees. \
-            The cars have Cuban license plates, indicating a location likely in Cuba."
-```
-
-**Referring Expression Segmentation**
-```bash
-python grounded_sam2_florence2_image_demo.py \
-    --pipeline referring_expression_segmentation \
-    --image_path ./notebooks/images/cars.jpg \
-    --text_input "The left red car."
-```
-
-**Open-Vocabulary Detection and Segmentation**
-```bash
-python grounded_sam2_florence2_image_demo.py \
-    --pipeline open_vocabulary_detection_segmentation \
-    --image_path ./notebooks/images/cars.jpg \
-    --text_input "car <and> building"
-```
-- Note that if you want to **detect multiple classes** you should split them with `<and>` in your input text.
-
-
-### Grounded SAM 2 Florence-2 Image Auto-Labeling Demo
-`Florence-2` can be used as a auto image annotator by cascading its caption capability with its grounding capability. 
-
-| Task | Task Prompt | Text Input |
-|:---:|:---:|:---:|
-| Caption + Phrase Grounding | `<CAPTION>` + `<CAPTION_TO_PHRASE_GROUNDING>` | &#10008; |
-| Detailed Caption + Phrase Grounding | `<DETAILED_CAPTION>` + `<CAPTION_TO_PHRASE_GROUNDING>` | &#10008; |
-| More Detailed Caption + Phrase Grounding | `<MORE_DETAILED_CAPTION>` + `<CAPTION_TO_PHRASE_GROUNDING>` | &#10008; |
-
-You can try the following scripts to run these demo:
-
-**Caption to Phrase Grounding**
-```bash
-python grounded_sam2_florence2_autolabel_pipeline.py \
-    --image_path ./notebooks/images/groceries.jpg \
-    --pipeline caption_to_phrase_grounding \
-    --caption_type caption
-```
-
-- You can specify `caption_type` to control the granularity of the caption, if you want a more detailed caption, you can try `--caption_type detailed_caption` or `--caption_type more_detailed_caption`.
-
-### Citation
-
-If you find this project helpful for your research, please consider citing the following BibTeX entry.
-
-```BibTex
-@misc{ravi2024sam2segmentimages,
-      title={SAM 2: Segment Anything in Images and Videos}, 
-      author={Nikhila Ravi and Valentin Gabeur and Yuan-Ting Hu and Ronghang Hu and Chaitanya Ryali and Tengyu Ma and Haitham Khedr and Roman Rädle and Chloe Rolland and Laura Gustafson and Eric Mintun and Junting Pan and Kalyan Vasudev Alwala and Nicolas Carion and Chao-Yuan Wu and Ross Girshick and Piotr Dollár and Christoph Feichtenhofer},
-      year={2024},
-      eprint={2408.00714},
-      archivePrefix={arXiv},
-      primaryClass={cs.CV},
-      url={https://arxiv.org/abs/2408.00714}, 
-}
-
-@article{liu2023grounding,
-  title={Grounding dino: Marrying dino with grounded pre-training for open-set object detection},
-  author={Liu, Shilong and Zeng, Zhaoyang and Ren, Tianhe and Li, Feng and Zhang, Hao and Yang, Jie and Li, Chunyuan and Yang, Jianwei and Su, Hang and Zhu, Jun and others},
-  journal={arXiv preprint arXiv:2303.05499},
-  year={2023}
-}
-
-@misc{ren2024grounding,
-      title={Grounding DINO 1.5: Advance the "Edge" of Open-Set Object Detection}, 
-      author={Tianhe Ren and Qing Jiang and Shilong Liu and Zhaoyang Zeng and Wenlong Liu and Han Gao and Hongjie Huang and Zhengyu Ma and Xiaoke Jiang and Yihao Chen and Yuda Xiong and Hao Zhang and Feng Li and Peijun Tang and Kent Yu and Lei Zhang},
-      year={2024},
-      eprint={2405.10300},
-      archivePrefix={arXiv},
-      primaryClass={cs.CV}
-}
-
-@misc{ren2024grounded,
-      title={Grounded SAM: Assembling Open-World Models for Diverse Visual Tasks}, 
-      author={Tianhe Ren and Shilong Liu and Ailing Zeng and Jing Lin and Kunchang Li and He Cao and Jiayu Chen and Xinyu Huang and Yukang Chen and Feng Yan and Zhaoyang Zeng and Hao Zhang and Feng Li and Jie Yang and Hongyang Li and Qing Jiang and Lei Zhang},
-      year={2024},
-      eprint={2401.14159},
-      archivePrefix={arXiv},
-      primaryClass={cs.CV}
-}
-
-@article{kirillov2023segany,
-  title={Segment Anything}, 
-  author={Kirillov, Alexander and Mintun, Eric and Ravi, Nikhila and Mao, Hanzi and Rolland, Chloe and Gustafson, Laura and Xiao, Tete and Whitehead, Spencer and Berg, Alexander C. and Lo, Wan-Yen and Doll{\'a}r, Piotr and Girshick, Ross},
-  journal={arXiv:2304.02643},
-  year={2023}
-}
-
-@misc{jiang2024trex2,
-      title={T-Rex2: Towards Generic Object Detection via Text-Visual Prompt Synergy}, 
-      author={Qing Jiang and Feng Li and Zhaoyang Zeng and Tianhe Ren and Shilong Liu and Lei Zhang},
-      year={2024},
-      eprint={2403.14610},
-      archivePrefix={arXiv},
-      primaryClass={cs.CV}
-}
+python grounded_samurai_dinox.py
 ```

SAM2_README.md

@@ -1,140 +0,0 @@
-# SAM 2: Segment Anything in Images and Videos
-
-**[AI at Meta, FAIR](https://ai.meta.com/research/)**
-
-[Nikhila Ravi](https://nikhilaravi.com/), [Valentin Gabeur](https://gabeur.github.io/), [Yuan-Ting Hu](https://scholar.google.com/citations?user=E8DVVYQAAAAJ&hl=en), [Ronghang Hu](https://ronghanghu.com/), [Chaitanya Ryali](https://scholar.google.com/citations?user=4LWx24UAAAAJ&hl=en), [Tengyu Ma](https://scholar.google.com/citations?user=VeTSl0wAAAAJ&hl=en), [Haitham Khedr](https://hkhedr.com/), [Roman Rädle](https://scholar.google.de/citations?user=Tpt57v0AAAAJ&hl=en), [Chloe Rolland](https://scholar.google.com/citations?hl=fr&user=n-SnMhoAAAAJ), [Laura Gustafson](https://scholar.google.com/citations?user=c8IpF9gAAAAJ&hl=en), [Eric Mintun](https://ericmintun.github.io/), [Junting Pan](https://junting.github.io/), [Kalyan Vasudev Alwala](https://scholar.google.co.in/citations?user=m34oaWEAAAAJ&hl=en), [Nicolas Carion](https://www.nicolascarion.com/), [Chao-Yuan Wu](https://chaoyuan.org/), [Ross Girshick](https://www.rossgirshick.info/), [Piotr Dollár](https://pdollar.github.io/), [Christoph Feichtenhofer](https://feichtenhofer.github.io/)
-
-[[`Paper`](https://ai.meta.com/research/publications/sam-2-segment-anything-in-images-and-videos/)] [[`Project`](https://ai.meta.com/sam2)] [[`Demo`](https://sam2.metademolab.com/)] [[`Dataset`](https://ai.meta.com/datasets/segment-anything-video)] [[`Blog`](https://ai.meta.com/blog/segment-anything-2)] [[`BibTeX`](#citing-sam-2)]
-
-![SAM 2 architecture](assets/model_diagram.png?raw=true)
-
-**Segment Anything Model 2 (SAM 2)** is a foundation model towards solving promptable visual segmentation in images and videos. We extend SAM to video by considering images as a video with a single frame. The model design is a simple transformer architecture with streaming memory for real-time video processing. We build a model-in-the-loop data engine, which improves model and data via user interaction, to collect [**our SA-V dataset**](https://ai.meta.com/datasets/segment-anything-video), the largest video segmentation dataset to date. SAM 2 trained on our data provides strong performance across a wide range of tasks and visual domains.
-
-![SA-V dataset](assets/sa_v_dataset.jpg?raw=true)
-
-## Installation
-
-Please install SAM 2 on a GPU machine using:
-
-```bash
-git clone https://github.com/facebookresearch/segment-anything-2.git
-
-cd segment-anything-2; pip install -e .
-```
-
-To use the SAM 2 predictor and run the example notebooks, `jupyter` and `matplotlib` are required and can be installed by:
-
-```bash
-pip install -e ".[demo]"
-```
-
-## Getting Started
-
-### Download Checkpoints
-
-First, we need to download a model checkpoint. All the model checkpoints can be downloaded by running:
-
-```bash
-cd checkpoints
-./download_ckpts.sh
-```
-
-or individually from:
-
-- [sam2_hiera_tiny.pt](https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_tiny.pt)
-- [sam2_hiera_small.pt](https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_small.pt)
-- [sam2_hiera_base_plus.pt](https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_base_plus.pt)
-- [sam2_hiera_large.pt](https://dl.fbaipublicfiles.com/segment_anything_2/072824/sam2_hiera_large.pt)
-
-Then SAM 2 can be used in a few lines as follows for image and video prediction.
-
-### Image prediction
-
-SAM 2 has all the capabilities of [SAM](https://github.com/facebookresearch/segment-anything) on static images, and we provide image prediction APIs that closely resemble SAM for image use cases. The `SAM2ImagePredictor` class has an easy interface for image prompting.
-
-```python
-import torch
-from sam2.build_sam import build_sam2
-from sam2.sam2_image_predictor import SAM2ImagePredictor
-
-checkpoint = "./checkpoints/sam2_hiera_large.pt"
-model_cfg = "sam2_hiera_l.yaml"
-predictor = SAM2ImagePredictor(build_sam2(model_cfg, checkpoint))
-
-with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16):
-    predictor.set_image(<your_image>)
-    masks, _, _ = predictor.predict(<input_prompts>)
-```
-
-Please refer to the examples in [image_predictor_example.ipynb](./notebooks/image_predictor_example.ipynb) for static image use cases.
-
-SAM 2 also supports automatic mask generation on images just like SAM. Please see [automatic_mask_generator_example.ipynb](./notebooks/automatic_mask_generator_example.ipynb) for automatic mask generation in images.
-
-### Video prediction
-
-For promptable segmentation and tracking in videos, we provide a video predictor with APIs for example to add prompts and propagate masklets throughout a video. SAM 2 supports video inference on multiple objects and uses an inference state to keep track of the interactions in each video.
-
-```python
-import torch
-from sam2.build_sam import build_sam2_video_predictor
-
-checkpoint = "./checkpoints/sam2_hiera_large.pt"
-model_cfg = "sam2_hiera_l.yaml"
-predictor = build_sam2_video_predictor(model_cfg, checkpoint)
-
-with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16):
-    state = predictor.init_state(<your_video>)
-
-    # add new prompts and instantly get the output on the same frame
-    frame_idx, object_ids, masks = predictor.add_new_points(state, <your prompts>):
-
-    # propagate the prompts to get masklets throughout the video
-    for frame_idx, object_ids, masks in predictor.propagate_in_video(state):
-        ...
-```
-
-Please refer to the examples in [video_predictor_example.ipynb](./notebooks/video_predictor_example.ipynb) for details on how to add prompts, make refinements, and track multiple objects in videos.
-
-## Model Description
-
-|      **Model**       | **Size (M)** |    **Speed (FPS)**     | **SA-V test (J&F)** | **MOSE val (J&F)** | **LVOS v2 (J&F)** |
-| :------------------: | :----------: | :--------------------: | :-----------------: | :----------------: | :---------------: |
-|   sam2_hiera_tiny    |     38.9     |          47.2          |        75.0         |        70.9        |       75.3        |
-|   sam2_hiera_small   |      46      | 43.3 (53.0 compiled\*) |        74.9         |        71.5        |       76.4        |
-| sam2_hiera_base_plus |     80.8     | 34.8 (43.8 compiled\*) |        74.7         |        72.8        |       75.8        |
-|   sam2_hiera_large   |    224.4     | 24.2 (30.2 compiled\*) |        76.0         |        74.6        |       79.8        |
-
-\* Compile the model by setting `compile_image_encoder: True` in the config.
-
-## Segment Anything Video Dataset
-
-See [sav_dataset/README.md](sav_dataset/README.md) for details.
-
-## License
-
-The models are licensed under the [Apache 2.0 license](./LICENSE). Please refer to our research paper for more details on the models.
-
-## Contributing
-
-See [contributing](CONTRIBUTING.md) and the [code of conduct](CODE_OF_CONDUCT.md).
-
-## Contributors
-
-The SAM 2 project was made possible with the help of many contributors (alphabetical):
-
-Karen Bergan, Daniel Bolya, Alex Bosenberg, Kai Brown, Vispi Cassod, Christopher Chedeau, Ida Cheng, Luc Dahlin, Shoubhik Debnath, Rene Martinez Doehner, Grant Gardner, Sahir Gomez, Rishi Godugu, Baishan Guo, Caleb Ho, Andrew Huang, Somya Jain, Bob Kamma, Amanda Kallet, Jake Kinney, Alexander Kirillov, Shiva Koduvayur, Devansh Kukreja, Robert Kuo, Aohan Lin, Parth Malani, Jitendra Malik, Mallika Malhotra, Miguel Martin, Alexander Miller, Sasha Mitts, William Ngan, George Orlin, Joelle Pineau, Kate Saenko, Rodrick Shepard, Azita Shokrpour, David Soofian, Jonathan Torres, Jenny Truong, Sagar Vaze, Meng Wang, Claudette Ward, Pengchuan Zhang.
-
-Third-party code: we use a GPU-based connected component algorithm adapted from [`cc_torch`](https://github.com/zsef123/Connected_components_PyTorch) (with its license in [`LICENSE_cctorch`](./LICENSE_cctorch)) as an optional post-processing step for the mask predictions.
-
-## Citing SAM 2
-
-If you use SAM 2 or the SA-V dataset in your research, please use the following BibTeX entry.
-
-```bibtex
-@article{ravi2024sam2,
-  title={SAM 2: Segment Anything in Images and Videos},
-  author={Ravi, Nikhila and Gabeur, Valentin and Hu, Yuan-Ting and Hu, Ronghang and Ryali, Chaitanya and Ma, Tengyu and Khedr, Haitham and R{\"a}dle, Roman and Rolland, Chloe and Gustafson, Laura and Mintun, Eric and Pan, Junting and Alwala, Kalyan Vasudev and Carion, Nicolas and Wu, Chao-Yuan and Girshick, Ross and Doll{\'a}r, Piotr and Feichtenhofer, Christoph},
-  journal={arXiv preprint},
-  year={2024}
-}
-```

SAMURAI_README.md

@@ -0,0 +1,139 @@
+<div align="center">
+<img align="left" width="100" height="100" src="https://github.com/user-attachments/assets/1834fc25-42ef-4237-9feb-53a01c137e83" alt="">
+
+# SAMURAI: Adapting Segment Anything Model for Zero-Shot Visual Tracking with Motion-Aware Memory
+
+[Cheng-Yen Yang](https://yangchris11.github.io), [Hsiang-Wei Huang](https://hsiangwei0903.github.io/), [Wenhao Chai](https://rese1f.github.io/), [Zhongyu Jiang](https://zhyjiang.github.io/#/), [Jenq-Neng Hwang](https://people.ece.uw.edu/hwang/)
+
+[Information Processing Lab, University of Washington](https://ipl-uw.github.io/) 
+</div>
+
+
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/samurai-adapting-segment-anything-model-for-1/visual-object-tracking-on-lasot-ext)](https://paperswithcode.com/sota/visual-object-tracking-on-lasot-ext?p=samurai-adapting-segment-anything-model-for-1)
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/samurai-adapting-segment-anything-model-for-1/visual-object-tracking-on-got-10k)](https://paperswithcode.com/sota/visual-object-tracking-on-got-10k?p=samurai-adapting-segment-anything-model-for-1)
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/samurai-adapting-segment-anything-model-for-1/visual-object-tracking-on-needforspeed)](https://paperswithcode.com/sota/visual-object-tracking-on-needforspeed?p=samurai-adapting-segment-anything-model-for-1)
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/samurai-adapting-segment-anything-model-for-1/visual-object-tracking-on-lasot)](https://paperswithcode.com/sota/visual-object-tracking-on-lasot?p=samurai-adapting-segment-anything-model-for-1)
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/samurai-adapting-segment-anything-model-for-1/visual-object-tracking-on-otb-2015)](https://paperswithcode.com/sota/visual-object-tracking-on-otb-2015?p=samurai-adapting-segment-anything-model-for-1)
+
+[[Arxiv]](https://arxiv.org/abs/2411.11922) [[Project Page]](https://yangchris11.github.io/samurai/) [[Raw Results]](https://drive.google.com/drive/folders/1ssiDmsC7mw5AiItYQG4poiR1JgRq305y?usp=sharing) 
+
+This repository is the official implementation of SAMURAI: Adapting Segment Anything Model for Zero-Shot Visual Tracking with Motion-Aware Memory
+
+https://github.com/user-attachments/assets/9d368ca7-2e9b-4fed-9da0-d2efbf620d88
+
+All rights are reserved to the copyright owners (TM & © Universal (2019)). This clip is not intended for commercial use and is solely for academic demonstration in a research paper. Original source can be found [here](https://www.youtube.com/watch?v=cwUzUzpG8aM&t=4s).
+
+## Getting Started
+
+#### SAMURAI Installation 
+
+SAM 2 needs to be installed first before use. The code requires `python>=3.10`, as well as `torch>=2.3.1` and `torchvision>=0.18.1`. Please follow the instructions [here](https://github.com/facebookresearch/sam2?tab=readme-ov-file) to install both PyTorch and TorchVision dependencies. You can install **the SAMURAI version** of SAM 2 on a GPU machine using:
+```
+cd sam2
+pip install -e .
+pip install -e ".[notebooks]"
+```
+
+Please see [INSTALL.md](https://github.com/facebookresearch/sam2/blob/main/INSTALL.md) from the original SAM 2 repository for FAQs on potential issues and solutions.
+
+Install other requirements:
+```
+pip install matplotlib==3.7 tikzplotlib jpeg4py opencv-python lmdb pandas scipy loguru
+```
+
+#### SAM 2.1 Checkpoint Download
+
+```
+cd checkpoints && \
+./download_ckpts.sh && \
+cd ..
+```
+
+#### Data Preparation
+
+Please prepare the data in the following format:
+```
+data/LaSOT
+├── airplane/
+│   ├── airplane-1/
+│   │   ├── full_occlusion.txt
+│   │   ├── groundtruth.txt
+│   │   ├── img
+│   │   ├── nlp.txt
+│   │   └── out_of_view.txt
+│   ├── airplane-2/
+│   ├── airplane-3/
+│   ├── ...
+├── basketball
+├── bear
+├── bicycle
+...
+├── training_set.txt
+└── testing_set.txt
+```
+
+#### Main Inference
+```
+python scripts/main_inference.py 
+```
+
+## Demo on Custom Video
+
+To run the demo with your custom video or frame directory, use the following examples:
+
+**Note:** The `.txt` file contains a single line with the bounding box of the first frame in `x,y,w,h` format.
+
+### Input is Video File
+
+```
+python scripts/demo.py --video_path <your_video.mp4> --txt_path <path_to_first_frame_bbox.txt>
+```
+
+### Input is Frame Folder
+```
+# Only JPG images are supported
+python scripts/demo.py --video_path <your_frame_directory> --txt_path <path_to_first_frame_bbox.txt>
+```
+
+## FAQs
+**Question 1:** Does SAMURAI need training? [issue 34](https://github.com/yangchris11/samurai/issues/34)
+
+**Answer 1:** Unlike real-life samurai, the proposed samurai do not require additional training. It is a zero-shot method, we directly use the weights from SAM 2.1 to conduct VOT experiments. Kalman filter is used to estimate the current and future state (bounding box location and scale in our case) of a moving object based on measurements over time, it is a common approach that had been adapt in the field of tracking for a long time which does not requires any training. Please refer to code for more detail.
+
+**Question 2:** Does SAMURAI support streaming input (e.g. webcam)?
+
+**Answer 2:** Not yet. The existing code doesn't support live/streaming video as we inherit most of the codebase from the amazing SAM 2. Some discussion that you might be interested in: facebookresearch/sam2#90, facebookresearch/sam2#388 (comment).
+
+**Question 3:** How to use SAMURAI in longer video?
+
+**Answer 3:** See the discussion from sam2 https://github.com/facebookresearch/sam2/issues/264.
+
+
+## Acknowledgment
+
+SAMURAI is built on top of [SAM 2](https://github.com/facebookresearch/sam2?tab=readme-ov-file) by Meta FAIR.
+
+The VOT evaluation code is modifed from [VOT Toolkit](https://github.com/votchallenge/toolkit) by Luka Čehovin Zajc.
+
+## Citation
+
+Please consider citing our paper and the wonderful `SAM 2` if you found our work interesting and useful.
+```
+@article{ravi2024sam2,
+  title={SAM 2: Segment Anything in Images and Videos},
+  author={Ravi, Nikhila and Gabeur, Valentin and Hu, Yuan-Ting and Hu, Ronghang and Ryali, Chaitanya and Ma, Tengyu and Khedr, Haitham and R{\"a}dle, Roman and Rolland, Chloe and Gustafson, Laura and Mintun, Eric and Pan, Junting and Alwala, Kalyan Vasudev and Carion, Nicolas and Wu, Chao-Yuan and Girshick, Ross and Doll{\'a}r, Piotr and Feichtenhofer, Christoph},
+  journal={arXiv preprint arXiv:2408.00714},
+  url={https://arxiv.org/abs/2408.00714},
+  year={2024}
+}
+
+@misc{yang2024samurai,
+      title={SAMURAI: Adapting Segment Anything Model for Zero-Shot Visual Tracking with Motion-Aware Memory}, 
+      author={Cheng-Yen Yang and Hsiang-Wei Huang and Wenhao Chai and Zhongyu Jiang and Jenq-Neng Hwang},
+      year={2024},
+      eprint={2411.11922},
+      archivePrefix={arXiv},
+      primaryClass={cs.CV},
+      url={https://arxiv.org/abs/2411.11922}, 
+}
+```

data/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

gdino_checkpoints/download_ckpts.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-
-# This source code is licensed under the license found in the
-# LICENSE file in the root directory of this source tree.
-
-
-# Define the URLs for the checkpoints
-BASE_URL="https://github.com/IDEA-Research/GroundingDINO/releases/download/"
-swint_ogc_url="${BASE_URL}v0.1.0-alpha/groundingdino_swint_ogc.pth"
-swinb_cogcoor_url="${BASE_URL}v0.1.0-alpha2/groundingdino_swinb_cogcoor.pth"
-
-
-
-# Download each of the four checkpoints using wget
-echo "Downloading groundingdino_swint_ogc.pth checkpoint..."
-wget $swint_ogc_url || { echo "Failed to download checkpoint from $swint_ogc_url"; exit 1; }
-
-echo "Downloading groundingdino_swinb_cogcoor.pth checkpoint..."
-wget $swinb_cogcoor_url || { echo "Failed to download checkpoint from $swinb_cogcoor_url"; exit 1; }
-
-echo "All checkpoints are downloaded successfully."

grounded_sam2_dinox_demo.py

@@ -1,245 +0,0 @@
-# dds cloudapi for Grounding DINO 1.5
-from dds_cloudapi_sdk import Config
-from dds_cloudapi_sdk import Client
-from dds_cloudapi_sdk.tasks.dinox import DinoxTask
-from dds_cloudapi_sdk.tasks.types import DetectionTarget
-from dds_cloudapi_sdk import TextPrompt
-
-import os
-import cv2
-import json
-import torch
-import tempfile
-import numpy as np
-import supervision as sv
-import pycocotools.mask as mask_util
-from pathlib import Path
-from PIL import Image
-from sam2.build_sam import build_sam2
-from sam2.sam2_image_predictor import SAM2ImagePredictor
-
-"""
-Hyper parameters
-"""
-API_TOKEN = "Your API token"
-TEXT_PROMPT = "car . building ."
-IMG_PATH = "notebooks/images/cars.jpg"
-SAM2_CHECKPOINT = "./checkpoints/sam2.1_hiera_large.pt"
-SAM2_MODEL_CONFIG = "configs/sam2.1/sam2.1_hiera_l.yaml"
-BOX_THRESHOLD = 0.2
-WITH_SLICE_INFERENCE = False
-SLICE_WH = (480, 480)
-OVERLAP_RATIO = (0.2, 0.2)
-DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
-OUTPUT_DIR = Path("outputs/grounded_sam2_dinox_demo")
-DUMP_JSON_RESULTS = True
-
-# create output directory
-OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
-
-"""
-Prompt DINO-X with Text for Box Prompt Generation with Cloud API
-"""
-# Step 1: initialize the config
-token = API_TOKEN
-config = Config(token)
-
-# Step 2: initialize the client
-client = Client(config)
-
-# Step 3: run the task by DetectionTask class
-# image_url = "https://algosplt.oss-cn-shenzhen.aliyuncs.com/test_files/tasks/detection/iron_man.jpg"
-# if you are processing local image file, upload them to DDS server to get the image url
-
-classes = [x.strip().lower() for x in TEXT_PROMPT.split('.') if x]
-class_name_to_id = {name: id for id, name in enumerate(classes)}
-class_id_to_name = {id: name for name, id in class_name_to_id.items()}
-
-if WITH_SLICE_INFERENCE:
-    def callback(image_slice: np.ndarray) -> sv.Detections:
-        print("Inference on image slice")
-        # save the img as temp img file for GD-1.5 API usage
-        with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as tmpfile:
-            temp_filename = tmpfile.name
-        cv2.imwrite(temp_filename, image_slice)
-        image_url = client.upload_file(temp_filename)
-        task = DinoxTask(
-            image_url=image_url,
-            prompts=[TextPrompt(text=TEXT_PROMPT)],
-            bbox_threshold=0.25,
-            targets=[DetectionTarget.BBox],
-        )
-        client.run_task(task)
-        result = task.result
-        # detele the tempfile
-        os.remove(temp_filename)
-        
-        input_boxes = []
-        confidences = []
-        class_ids = []
-        objects = result.objects
-        for idx, obj in enumerate(objects):
-            input_boxes.append(obj.bbox)
-            confidences.append(obj.score)
-            cls_name = obj.category.lower().strip()
-            class_ids.append(class_name_to_id[cls_name])
-        # ensure input_boxes with shape (_, 4)
-        input_boxes = np.array(input_boxes).reshape(-1, 4)
-        class_ids = np.array(class_ids)
-        confidences = np.array(confidences)
-        return sv.Detections(xyxy=input_boxes, confidence=confidences, class_id=class_ids)
-    
-    slicer = sv.InferenceSlicer(
-        callback=callback,
-        slice_wh=SLICE_WH,
-        overlap_ratio_wh=OVERLAP_RATIO,
-        iou_threshold=0.5,
-        overlap_filter_strategy=sv.OverlapFilter.NON_MAX_SUPPRESSION
-        )
-    detections = slicer(cv2.imread(IMG_PATH))
-    class_names = [class_id_to_name[id] for id in detections.class_id]
-    confidences = detections.confidence
-    class_ids = detections.class_id
-    input_boxes = detections.xyxy
-else:
-    image_url = client.upload_file(IMG_PATH)
-
-    task = DinoxTask(
-        image_url=image_url,
-        prompts=[TextPrompt(text=TEXT_PROMPT)],
-        bbox_threshold=0.25,
-        targets=[DetectionTarget.BBox],
-    )
-
-    client.run_task(task)
-    result = task.result
-
-    objects = result.objects  # the list of detected objects
-
-
-    input_boxes = []
-    confidences = []
-    class_names = []
-    class_ids = []
-
-    for idx, obj in enumerate(objects):
-        input_boxes.append(obj.bbox)
-        confidences.append(obj.score)
-        cls_name = obj.category.lower().strip()
-        class_names.append(cls_name)
-        class_ids.append(class_name_to_id[cls_name])
-
-    input_boxes = np.array(input_boxes)
-    class_ids = np.array(class_ids)
-
-"""
-Init SAM 2 Model and Predict Mask with Box Prompt
-"""
-
-# environment settings
-# use bfloat16
-torch.autocast(device_type=DEVICE, dtype=torch.bfloat16).__enter__()
-
-if torch.cuda.get_device_properties(0).major >= 8:
-    # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)
-    torch.backends.cuda.matmul.allow_tf32 = True
-    torch.backends.cudnn.allow_tf32 = True
-
-# build SAM2 image predictor
-sam2_checkpoint = SAM2_CHECKPOINT
-model_cfg = SAM2_MODEL_CONFIG
-sam2_model = build_sam2(model_cfg, sam2_checkpoint, device=DEVICE)
-sam2_predictor = SAM2ImagePredictor(sam2_model)
-
-image = Image.open(IMG_PATH)
-
-sam2_predictor.set_image(np.array(image.convert("RGB")))
-
-masks, scores, logits = sam2_predictor.predict(
-    point_coords=None,
-    point_labels=None,
-    box=input_boxes,
-    multimask_output=False,
-)
-
-
-"""
-Post-process the output of the model to get the masks, scores, and logits for visualization
-"""
-# convert the shape to (n, H, W)
-if masks.ndim == 4:
-    masks = masks.squeeze(1)
-
-
-"""
-Visualization the Predict Results
-"""
-
-labels = [
-    f"{class_name} {confidence:.2f}"
-    for class_name, confidence
-    in zip(class_names, confidences)
-]
-
-"""
-Visualize image with supervision useful API
-"""
-img = cv2.imread(IMG_PATH)
-detections = sv.Detections(
-    xyxy=input_boxes,  # (n, 4)
-    mask=masks.astype(bool),  # (n, h, w)
-    class_id=class_ids
-)
-
-box_annotator = sv.BoxAnnotator()
-annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-
-label_annotator = sv.LabelAnnotator()
-annotated_frame = label_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels)
-cv2.imwrite(os.path.join(OUTPUT_DIR, "dinox_annotated_image.jpg"), annotated_frame)
-
-mask_annotator = sv.MaskAnnotator()
-annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-cv2.imwrite(os.path.join(OUTPUT_DIR, "dinox_sam2_annotated_image_with_mask.jpg"), annotated_frame)
-
-print(f'Annotated image has already been saved as to "{OUTPUT_DIR}"')
-
-"""
-Dump the results in standard format and save as json files
-"""
-
-def single_mask_to_rle(mask):
-    rle = mask_util.encode(np.array(mask[:, :, None], order="F", dtype="uint8"))[0]
-    rle["counts"] = rle["counts"].decode("utf-8")
-    return rle
-
-if DUMP_JSON_RESULTS:
-    print("Start dumping the annotation...")
-    # convert mask into rle format
-    mask_rles = [single_mask_to_rle(mask) for mask in masks]
-
-    input_boxes = input_boxes.tolist()
-    scores = scores.tolist()
-    # FIXME: class_names should be a list of strings without spaces
-    class_names = [class_name.strip() for class_name in class_names]
-    # save the results in standard format
-    results = {
-        "image_path": IMG_PATH,
-        "annotations" : [
-            {
-                "class_name": class_name,
-                "bbox": box,
-                "segmentation": mask_rle,
-                "score": score,
-            }
-            for class_name, box, mask_rle, score in zip(class_names, input_boxes, mask_rles, scores)
-        ],
-        "box_format": "xyxy",
-        "img_width": image.width,
-        "img_height": image.height,
-    }
-    
-    with open(os.path.join(OUTPUT_DIR, "grounded_sam2_dinox_image_demo_results.json"), "w") as f:
-        json.dump(results, f, indent=4)
-
-    print(f'Annotation has already been saved to "{OUTPUT_DIR}"')

grounded_sam2_florence2_autolabel_pipeline.py

@@ -1,198 +0,0 @@
-import os
-import cv2
-import torch
-import argparse
-import numpy as np
-import supervision as sv
-from PIL import Image
-from sam2.build_sam import build_sam2
-from sam2.sam2_image_predictor import SAM2ImagePredictor
-from transformers import AutoProcessor, AutoModelForCausalLM
-from utils.supervision_utils import CUSTOM_COLOR_MAP
-
-"""
-Define Some Hyperparam
-"""
-
-TASK_PROMPT = {
-    "caption": "<CAPTION>",
-    "detailed_caption": "<DETAILED_CAPTION>",
-    "more_detailed_caption": "<MORE_DETAILED_CAPTION>",
-    "object_detection": "<OD>",
-    "dense_region_caption": "<DENSE_REGION_CAPTION>",
-    "region_proposal": "<REGION_PROPOSAL>",
-    "phrase_grounding": "<CAPTION_TO_PHRASE_GROUNDING>",
-    "referring_expression_segmentation": "<REFERRING_EXPRESSION_SEGMENTATION>",
-    "region_to_segmentation": "<REGION_TO_SEGMENTATION>",
-    "open_vocabulary_detection": "<OPEN_VOCABULARY_DETECTION>",
-    "region_to_category": "<REGION_TO_CATEGORY>",
-    "region_to_description": "<REGION_TO_DESCRIPTION>",
-    "ocr": "<OCR>",
-    "ocr_with_region": "<OCR_WITH_REGION>",
-}
-
-OUTPUT_DIR = "./outputs"
-
-if not os.path.exists(OUTPUT_DIR):
-    os.makedirs(OUTPUT_DIR, exist_ok=True)
-
-"""
-Init Florence-2 and SAM 2 Model
-"""
-
-FLORENCE2_MODEL_ID = "microsoft/Florence-2-large"
-SAM2_CHECKPOINT = "./checkpoints/sam2_hiera_large.pt"
-SAM2_CONFIG = "sam2_hiera_l.yaml"
-
-# environment settings
-# use bfloat16
-torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__()
-
-if torch.cuda.get_device_properties(0).major >= 8:
-    # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)
-    torch.backends.cuda.matmul.allow_tf32 = True
-    torch.backends.cudnn.allow_tf32 = True
-
-device = "cuda:0" if torch.cuda.is_available() else "cpu"
-torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
-
-# build florence-2
-florence2_model = AutoModelForCausalLM.from_pretrained(FLORENCE2_MODEL_ID, trust_remote_code=True, torch_dtype='auto').eval().to(device)
-florence2_processor = AutoProcessor.from_pretrained(FLORENCE2_MODEL_ID, trust_remote_code=True)
-
-# build sam 2
-sam2_model = build_sam2(SAM2_CONFIG, SAM2_CHECKPOINT, device=device)
-sam2_predictor = SAM2ImagePredictor(sam2_model)
-
-def run_florence2(task_prompt, text_input, model, processor, image):
-    assert model is not None, "You should pass the init florence-2 model here"
-    assert processor is not None, "You should set florence-2 processor here"
-
-    device = model.device
-
-    if text_input is None:
-        prompt = task_prompt
-    else:
-        prompt = task_prompt + text_input
-    
-    inputs = processor(text=prompt, images=image, return_tensors="pt").to(device, torch.float16)
-    generated_ids = model.generate(
-      input_ids=inputs["input_ids"].to(device),
-      pixel_values=inputs["pixel_values"].to(device),
-      max_new_tokens=1024,
-      early_stopping=False,
-      do_sample=False,
-      num_beams=3,
-    )
-    generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
-    parsed_answer = processor.post_process_generation(
-        generated_text, 
-        task=task_prompt, 
-        image_size=(image.width, image.height)
-    )
-    return parsed_answer
-
-
-"""
-We try to support a series of cascaded auto-labelling pipelines with Florence-2 and SAM 2
-"""
-
-"""
-Auto-Labelling Pipeline 1: Caption/Detailed Caption/More Detailed Caption + Phrase Grounding + Segmentation
-"""
-def caption_phrase_grounding_and_segmentation(
-    florence2_model,
-    florence2_processor,
-    sam2_predictor,
-    image_path,
-    caption_task_prompt='<CAPTION>',
-    output_dir=OUTPUT_DIR
-):
-    assert caption_task_prompt in ["<CAPTION>", "<DETAILED_CAPTION>", "<MORE_DETAILED_CAPTION>"]
-    image = Image.open(image_path).convert("RGB")
-    
-    # image caption
-    caption_results = run_florence2(caption_task_prompt, None, florence2_model, florence2_processor, image)
-    text_input = caption_results[caption_task_prompt]
-    print(f'Image caption for "{image_path}": ', text_input)
-    
-    # phrase grounding
-    grounding_results = run_florence2('<CAPTION_TO_PHRASE_GROUNDING>', text_input, florence2_model, florence2_processor, image)
-    grounding_results = grounding_results['<CAPTION_TO_PHRASE_GROUNDING>']
-    
-    # parse florence-2 detection results
-    input_boxes = np.array(grounding_results["bboxes"])
-    class_names = grounding_results["labels"]
-    class_ids = np.array(list(range(len(class_names))))
-    
-    # predict mask with SAM 2
-    sam2_predictor.set_image(np.array(image))
-    masks, scores, logits = sam2_predictor.predict(
-        point_coords=None,
-        point_labels=None,
-        box=input_boxes,
-        multimask_output=False,
-    )
-    
-    if masks.ndim == 4:
-        masks = masks.squeeze(1)
-    
-    # specify labels
-    labels = [
-        f"{class_name}" for class_name in class_names
-    ]
-    
-    # visualization results
-    img = cv2.imread(image_path)
-    detections = sv.Detections(
-        xyxy=input_boxes,
-        mask=masks.astype(bool),
-        class_id=class_ids
-    )
-    
-    box_annotator = sv.BoxAnnotator()
-    annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-    
-    label_annotator = sv.LabelAnnotator()
-    annotated_frame = label_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels)
-    cv2.imwrite(os.path.join(output_dir, "grounded_sam2_florence2_auto_labelling.jpg"), annotated_frame)
-    
-    mask_annotator = sv.MaskAnnotator()
-    annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-    cv2.imwrite(os.path.join(output_dir, "grounded_sam2_florence2_auto_labelling_with_mask.jpg"), annotated_frame)
-
-    print(f'Successfully save annotated image to "{output_dir}"')
-
-
-if __name__ == "__main__":
-
-    parser = argparse.ArgumentParser("Grounded SAM 2 Florence-2 Demos", add_help=True)
-    parser.add_argument("--image_path", type=str, default="./notebooks/images/cars.jpg", required=True, help="path to image file")
-    parser.add_argument("--pipeline", type=str, default="caption_to_phrase_grounding", required=True, help="pipeline to use")
-    parser.add_argument("--caption_type", type=str, default="caption", required=False, help="granularity of caption")
-    args = parser.parse_args()
-
-    CAPTION_TO_TASK_PROMPT = {
-        "caption": "<CAPTION>",
-        "detailed_caption": "<DETAILED_CAPTION>",
-        "more_detailed_caption": "<MORE_DETAILED_CAPTION>"
-    }
-
-    IMAGE_PATH = args.image_path
-    PIPELINE = args.pipeline
-    CAPTION_TYPE = args.caption_type
-    assert CAPTION_TYPE in ["caption", "detailed_caption", "more_detailed_caption"]
-    
-    print(f"Running pipeline: {PIPELINE} now.")
-
-    if PIPELINE == "caption_to_phrase_grounding":
-        # pipeline-1: caption + phrase grounding + segmentation
-        caption_phrase_grounding_and_segmentation(
-            florence2_model=florence2_model,
-            florence2_processor=florence2_processor,
-            sam2_predictor=sam2_predictor,
-            caption_task_prompt=CAPTION_TO_TASK_PROMPT[CAPTION_TYPE],
-            image_path=IMAGE_PATH
-        )
-    else:
-        raise NotImplementedError(f"Pipeline: {args.pipeline} is not implemented at this time")

grounded_sam2_florence2_image_demo.py

@@ -1,657 +0,0 @@
-import os
-import cv2
-import torch
-import argparse
-import numpy as np
-import supervision as sv
-from PIL import Image
-from sam2.build_sam import build_sam2
-from sam2.sam2_image_predictor import SAM2ImagePredictor
-from transformers import AutoProcessor, AutoModelForCausalLM
-from utils.supervision_utils import CUSTOM_COLOR_MAP
-
-"""
-Define Some Hyperparam
-"""
-
-TASK_PROMPT = {
-    "caption": "<CAPTION>",
-    "detailed_caption": "<DETAILED_CAPTION>",
-    "more_detailed_caption": "<MORE_DETAILED_CAPTION",
-    "object_detection": "<OD>",
-    "dense_region_caption": "<DENSE_REGION_CAPTION>",
-    "region_proposal": "<REGION_PROPOSAL>",
-    "phrase_grounding": "<CAPTION_TO_PHRASE_GROUNDING>",
-    "referring_expression_segmentation": "<REFERRING_EXPRESSION_SEGMENTATION>",
-    "region_to_segmentation": "<REGION_TO_SEGMENTATION>",
-    "open_vocabulary_detection": "<OPEN_VOCABULARY_DETECTION>",
-    "region_to_category": "<REGION_TO_CATEGORY>",
-    "region_to_description": "<REGION_TO_DESCRIPTION>",
-    "ocr": "<OCR>",
-    "ocr_with_region": "<OCR_WITH_REGION>",
-}
-
-OUTPUT_DIR = "./outputs"
-
-if not os.path.exists(OUTPUT_DIR):
-    os.makedirs(OUTPUT_DIR, exist_ok=True)
-
-"""
-Init Florence-2 and SAM 2 Model
-"""
-
-FLORENCE2_MODEL_ID = "microsoft/Florence-2-large"
-SAM2_CHECKPOINT = "./checkpoints/sam2.1_hiera_large.pt"
-SAM2_CONFIG = "configs/sam2.1/sam2.1_hiera_l.yaml"
-
-# environment settings
-# use bfloat16
-torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__()
-
-if torch.cuda.get_device_properties(0).major >= 8:
-    # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)
-    torch.backends.cuda.matmul.allow_tf32 = True
-    torch.backends.cudnn.allow_tf32 = True
-
-device = "cuda:0" if torch.cuda.is_available() else "cpu"
-torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
-
-# build florence-2
-florence2_model = AutoModelForCausalLM.from_pretrained(FLORENCE2_MODEL_ID, trust_remote_code=True, torch_dtype='auto').eval().to(device)
-florence2_processor = AutoProcessor.from_pretrained(FLORENCE2_MODEL_ID, trust_remote_code=True)
-
-# build sam 2
-sam2_model = build_sam2(SAM2_CONFIG, SAM2_CHECKPOINT, device=device)
-sam2_predictor = SAM2ImagePredictor(sam2_model)
-
-def run_florence2(task_prompt, text_input, model, processor, image):
-    assert model is not None, "You should pass the init florence-2 model here"
-    assert processor is not None, "You should set florence-2 processor here"
-
-    device = model.device
-
-    if text_input is None:
-        prompt = task_prompt
-    else:
-        prompt = task_prompt + text_input
-    
-    inputs = processor(text=prompt, images=image, return_tensors="pt").to(device, torch.float16)
-    generated_ids = model.generate(
-      input_ids=inputs["input_ids"].to(device),
-      pixel_values=inputs["pixel_values"].to(device),
-      max_new_tokens=1024,
-      early_stopping=False,
-      do_sample=False,
-      num_beams=3,
-    )
-    generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
-    parsed_answer = processor.post_process_generation(
-        generated_text, 
-        task=task_prompt, 
-        image_size=(image.width, image.height)
-    )
-    return parsed_answer
-
-
-"""
-We support a set of pipelines built by Florence-2 + SAM 2
-"""
-
-"""
-Pipeline-1: Object Detection + Segmentation
-"""
-def object_detection_and_segmentation(
-    florence2_model,
-    florence2_processor,
-    sam2_predictor,
-    image_path,
-    task_prompt="<OD>",
-    text_input=None,
-    output_dir=OUTPUT_DIR
-):
-    assert text_input is None, "Text input should be None when calling object detection pipeline."
-    # run florence-2 object detection in demo
-    image = Image.open(image_path).convert("RGB")
-    results = run_florence2(task_prompt, text_input, florence2_model, florence2_processor, image)
-    
-    """ Florence-2 Object Detection Output Format
-    {'<OD>': 
-        {
-            'bboxes': 
-                [
-                    [33.599998474121094, 159.59999084472656, 596.7999877929688, 371.7599792480469], 
-                    [454.0799865722656, 96.23999786376953, 580.7999877929688, 261.8399963378906], 
-                    [224.95999145507812, 86.15999603271484, 333.7599792480469, 164.39999389648438], 
-                    [449.5999755859375, 276.239990234375, 554.5599975585938, 370.3199768066406], 
-                    [91.19999694824219, 280.0799865722656, 198.0800018310547, 370.3199768066406]
-                ], 
-            'labels': ['car', 'door', 'door', 'wheel', 'wheel']
-        }
-    }
-    """
-    results = results[task_prompt]
-    # parse florence-2 detection results
-    input_boxes = np.array(results["bboxes"])
-    class_names = results["labels"]
-    class_ids = np.array(list(range(len(class_names))))
-    
-    # predict mask with SAM 2
-    sam2_predictor.set_image(np.array(image))
-    masks, scores, logits = sam2_predictor.predict(
-        point_coords=None,
-        point_labels=None,
-        box=input_boxes,
-        multimask_output=False,
-    )
-    
-    if masks.ndim == 4:
-        masks = masks.squeeze(1)
-    
-    # specify labels
-    labels = [
-        f"{class_name}" for class_name in class_names
-    ]
-    
-    # visualization results
-    img = cv2.imread(image_path)
-    detections = sv.Detections(
-        xyxy=input_boxes,
-        mask=masks.astype(bool),
-        class_id=class_ids
-    )
-    
-    box_annotator = sv.BoxAnnotator()
-    annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-    
-    label_annotator = sv.LabelAnnotator()
-    annotated_frame = label_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels)
-    cv2.imwrite(os.path.join(output_dir, "grounded_sam2_florence2_det_annotated_image.jpg"), annotated_frame)
-    
-    mask_annotator = sv.MaskAnnotator()
-    annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-    cv2.imwrite(os.path.join(output_dir, "grounded_sam2_florence2_det_image_with_mask.jpg"), annotated_frame)
-
-    print(f'Successfully save annotated image to "{output_dir}"')
-
-"""
-Pipeline 2: Dense Region Caption + Segmentation
-"""
-def dense_region_caption_and_segmentation(
-    florence2_model,
-    florence2_processor,
-    sam2_predictor,
-    image_path,
-    task_prompt="<DENSE_REGION_CAPTION>",
-    text_input=None,
-    output_dir=OUTPUT_DIR
-):
-    assert text_input is None, "Text input should be None when calling dense region caption pipeline."
-    # run florence-2 object detection in demo
-    image = Image.open(image_path).convert("RGB")
-    results = run_florence2(task_prompt, text_input, florence2_model, florence2_processor, image)
-    
-    """ Florence-2 Object Detection Output Format
-    {'<DENSE_REGION_CAPTION>': 
-        {
-            'bboxes': 
-                [
-                    [33.599998474121094, 159.59999084472656, 596.7999877929688, 371.7599792480469], 
-                    [454.0799865722656, 96.23999786376953, 580.7999877929688, 261.8399963378906], 
-                    [224.95999145507812, 86.15999603271484, 333.7599792480469, 164.39999389648438], 
-                    [449.5999755859375, 276.239990234375, 554.5599975585938, 370.3199768066406], 
-                    [91.19999694824219, 280.0799865722656, 198.0800018310547, 370.3199768066406]
-                ], 
-            'labels': ['turquoise Volkswagen Beetle', 'wooden double doors with metal handles', 'wheel', 'wheel', 'door']
-        }
-    }
-    """
-    results = results[task_prompt]
-    # parse florence-2 detection results
-    input_boxes = np.array(results["bboxes"])
-    class_names = results["labels"]
-    class_ids = np.array(list(range(len(class_names))))
-    
-    # predict mask with SAM 2
-    sam2_predictor.set_image(np.array(image))
-    masks, scores, logits = sam2_predictor.predict(
-        point_coords=None,
-        point_labels=None,
-        box=input_boxes,
-        multimask_output=False,
-    )
-    
-    if masks.ndim == 4:
-        masks = masks.squeeze(1)
-    
-    # specify labels
-    labels = [
-        f"{class_name}" for class_name in class_names
-    ]
-    
-    # visualization results
-    img = cv2.imread(image_path)
-    detections = sv.Detections(
-        xyxy=input_boxes,
-        mask=masks.astype(bool),
-        class_id=class_ids
-    )
-    
-    box_annotator = sv.BoxAnnotator()
-    annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-    
-    label_annotator = sv.LabelAnnotator()
-    annotated_frame = label_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels)
-    cv2.imwrite(os.path.join(output_dir, "grounded_sam2_florence2_dense_region_cap_annotated_image.jpg"), annotated_frame)
-    
-    mask_annotator = sv.MaskAnnotator()
-    annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-    cv2.imwrite(os.path.join(output_dir, "grounded_sam2_florence2_dense_region_cap_image_with_mask.jpg"), annotated_frame)
-
-    print(f'Successfully save annotated image to "{output_dir}"')
-
-
-"""
-Pipeline 3: Region Proposal + Segmentation
-"""
-def region_proposal_and_segmentation(
-    florence2_model,
-    florence2_processor,
-    sam2_predictor,
-    image_path,
-    task_prompt="<REGION_PROPOSAL>",
-    text_input=None,
-    output_dir=OUTPUT_DIR
-):
-    assert text_input is None, "Text input should be None when calling region proposal pipeline."
-    # run florence-2 object detection in demo
-    image = Image.open(image_path).convert("RGB")
-    results = run_florence2(task_prompt, text_input, florence2_model, florence2_processor, image)
-    
-    """ Florence-2 Object Detection Output Format
-    {'<REGION_PROPOSAL>': 
-        {
-            'bboxes': 
-                [
-                    [33.599998474121094, 159.59999084472656, 596.7999877929688, 371.7599792480469], 
-                    [454.0799865722656, 96.23999786376953, 580.7999877929688, 261.8399963378906], 
-                    [224.95999145507812, 86.15999603271484, 333.7599792480469, 164.39999389648438], 
-                    [449.5999755859375, 276.239990234375, 554.5599975585938, 370.3199768066406], 
-                    [91.19999694824219, 280.0799865722656, 198.0800018310547, 370.3199768066406]
-                ], 
-            'labels': ['', '', '', '', '', '', '']
-        }
-    }
-    """
-    results = results[task_prompt]
-    # parse florence-2 detection results
-    input_boxes = np.array(results["bboxes"])
-    class_names = results["labels"]
-    class_ids = np.array(list(range(len(class_names))))
-    
-    # predict mask with SAM 2
-    sam2_predictor.set_image(np.array(image))
-    masks, scores, logits = sam2_predictor.predict(
-        point_coords=None,
-        point_labels=None,
-        box=input_boxes,
-        multimask_output=False,
-    )
-    
-    if masks.ndim == 4:
-        masks = masks.squeeze(1)
-    
-    # specify labels
-    labels = [
-        f"region_{idx}" for idx, class_name in enumerate(class_names)
-    ]
-    
-    # visualization results
-    img = cv2.imread(image_path)
-    detections = sv.Detections(
-        xyxy=input_boxes,
-        mask=masks.astype(bool),
-        class_id=class_ids
-    )
-    
-    box_annotator = sv.BoxAnnotator()
-    annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-    
-    label_annotator = sv.LabelAnnotator()
-    annotated_frame = label_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels)
-    cv2.imwrite(os.path.join(output_dir, "grounded_sam2_florence2_region_proposal.jpg"), annotated_frame)
-    
-    mask_annotator = sv.MaskAnnotator()
-    annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-    cv2.imwrite(os.path.join(output_dir, "grounded_sam2_florence2_region_proposal_with_mask.jpg"), annotated_frame)
-
-    print(f'Successfully save annotated image to "{output_dir}"')
-
-
-"""
-Pipeline 4: Phrase Grounding + Segmentation
-"""
-def phrase_grounding_and_segmentation(
-    florence2_model,
-    florence2_processor,
-    sam2_predictor,
-    image_path,
-    task_prompt="<CAPTION_TO_PHRASE_GROUNDING>",
-    text_input=None,
-    output_dir=OUTPUT_DIR
-):
-    # run florence-2 object detection in demo
-    image = Image.open(image_path).convert("RGB")
-    results = run_florence2(task_prompt, text_input, florence2_model, florence2_processor, image)
-    
-    """ Florence-2 Object Detection Output Format
-    {'<CAPTION_TO_PHRASE_GROUNDING>': 
-        {
-            'bboxes': 
-                [
-                    [34.23999786376953, 159.1199951171875, 582.0800170898438, 374.6399841308594], 
-                    [1.5999999046325684, 4.079999923706055, 639.0399780273438, 305.03997802734375]
-                ], 
-            'labels': ['A green car', 'a yellow building']
-        }
-    }
-    """
-    assert text_input is not None, "Text input should not be None when calling phrase grounding pipeline."
-    results = results[task_prompt]
-    # parse florence-2 detection results
-    input_boxes = np.array(results["bboxes"])
-    class_names = results["labels"]
-    class_ids = np.array(list(range(len(class_names))))
-    
-    # predict mask with SAM 2
-    sam2_predictor.set_image(np.array(image))
-    masks, scores, logits = sam2_predictor.predict(
-        point_coords=None,
-        point_labels=None,
-        box=input_boxes,
-        multimask_output=False,
-    )
-    
-    if masks.ndim == 4:
-        masks = masks.squeeze(1)
-    
-    # specify labels
-    labels = [
-        f"{class_name}" for class_name in class_names
-    ]
-    
-    # visualization results
-    img = cv2.imread(image_path)
-    detections = sv.Detections(
-        xyxy=input_boxes,
-        mask=masks.astype(bool),
-        class_id=class_ids
-    )
-    
-    box_annotator = sv.BoxAnnotator()
-    annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-    
-    label_annotator = sv.LabelAnnotator()
-    annotated_frame = label_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels)
-    cv2.imwrite(os.path.join(output_dir, "grounded_sam2_florence2_phrase_grounding.jpg"), annotated_frame)
-    
-    mask_annotator = sv.MaskAnnotator()
-    annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-    cv2.imwrite(os.path.join(output_dir, "grounded_sam2_florence2_phrase_grounding_with_mask.jpg"), annotated_frame)
-
-    print(f'Successfully save annotated image to "{output_dir}"')
-
-
-"""
-Pipeline 5: Referring Expression Segmentation
-
-Note that Florence-2 directly support referring segmentation with polygon output format, which may be not that accurate, 
-therefore we try to decode box from polygon and use SAM 2 for mask prediction
-"""
-def referring_expression_segmentation(
-    florence2_model,
-    florence2_processor,
-    sam2_predictor,
-    image_path,
-    task_prompt="<REFERRING_EXPRESSION_SEGMENTATION>",
-    text_input=None,
-    output_dir=OUTPUT_DIR
-):
-    # run florence-2 object detection in demo
-    image = Image.open(image_path).convert("RGB")
-    results = run_florence2(task_prompt, text_input, florence2_model, florence2_processor, image)
-    
-    """ Florence-2 Object Detection Output Format
-    {'<REFERRING_EXPRESSION_SEGMENTATION>': 
-        {
-            'polygons': [[[...]]]
-            'labels': ['']
-        }
-    }
-    """
-    assert text_input is not None, "Text input should not be None when calling referring segmentation pipeline."
-    results = results[task_prompt]
-    # parse florence-2 detection results
-    polygon_points = np.array(results["polygons"][0], dtype=np.int32).reshape(-1, 2)
-    class_names = [text_input]
-    class_ids = np.array(list(range(len(class_names))))
-    
-    # parse polygon format to mask
-    img_width, img_height = image.size[0], image.size[1]
-    florence2_mask = np.zeros((img_height, img_width), dtype=np.uint8)
-    if len(polygon_points) < 3:
-        print("Invalid polygon:", polygon_points)
-        exit()
-    cv2.fillPoly(florence2_mask, [polygon_points], 1)
-    if florence2_mask.ndim == 2:
-        florence2_mask = florence2_mask[None]
-
-    # compute bounding box based on polygon points
-    x_min = np.min(polygon_points[:, 0])
-    y_min = np.min(polygon_points[:, 1])
-    x_max = np.max(polygon_points[:, 0])
-    y_max = np.max(polygon_points[:, 1])
-
-    input_boxes = np.array([[x_min, y_min, x_max, y_max]])
-
-    # predict mask with SAM 2
-    sam2_predictor.set_image(np.array(image))
-    sam2_masks, scores, logits = sam2_predictor.predict(
-        point_coords=None,
-        point_labels=None,
-        box=input_boxes,
-        multimask_output=False,
-    )
-    
-    if sam2_masks.ndim == 4:
-        sam2_masks = sam2_masks.squeeze(1)
-    
-    # specify labels
-    labels = [
-        f"{class_name}" for class_name in class_names
-    ]
-    
-    # visualization florence2 mask
-    img = cv2.imread(image_path)
-    detections = sv.Detections(
-        xyxy=input_boxes,
-        mask=florence2_mask.astype(bool),
-        class_id=class_ids
-    )
-    
-    box_annotator = sv.BoxAnnotator()
-    annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-    
-    label_annotator = sv.LabelAnnotator()
-    annotated_frame = label_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels)
-    cv2.imwrite(os.path.join(output_dir, "florence2_referring_segmentation_box.jpg"), annotated_frame)
-    
-    mask_annotator = sv.MaskAnnotator()
-    annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-    cv2.imwrite(os.path.join(output_dir, "florence2_referring_segmentation_box_with_mask.jpg"), annotated_frame)
-
-    print(f'Successfully save florence-2 annotated image to "{output_dir}"')
-
-    # visualize sam2 mask
-    img = cv2.imread(image_path)
-    detections = sv.Detections(
-        xyxy=input_boxes,
-        mask=sam2_masks.astype(bool),
-        class_id=class_ids
-    )
-
-    box_annotator = sv.BoxAnnotator()
-    annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-    
-    label_annotator = sv.LabelAnnotator()
-    annotated_frame = label_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels)
-    cv2.imwrite(os.path.join(output_dir, "grounded_sam2_florence2_referring_box.jpg"), annotated_frame)
-    
-    mask_annotator = sv.MaskAnnotator()
-    annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-    cv2.imwrite(os.path.join(output_dir, "grounded_sam2_florence2_referring_box_with_sam2_mask.jpg"), annotated_frame)
-
-    print(f'Successfully save sam2 annotated image to "{output_dir}"')
-
-
-"""
-Pipeline 6: Open-Vocabulary Detection + Segmentation
-"""
-def open_vocabulary_detection_and_segmentation(
-    florence2_model,
-    florence2_processor,
-    sam2_predictor,
-    image_path,
-    task_prompt="<OPEN_VOCABULARY_DETECTION>",
-    text_input=None,
-    output_dir=OUTPUT_DIR
-):
-    # run florence-2 object detection in demo
-    image = Image.open(image_path).convert("RGB")
-    results = run_florence2(task_prompt, text_input, florence2_model, florence2_processor, image)
-    
-    """ Florence-2 Open-Vocabulary Detection Output Format
-    {'<OPEN_VOCABULARY_DETECTION>': 
-        {
-            'bboxes': 
-                [
-                    [34.23999786376953, 159.1199951171875, 582.0800170898438, 374.6399841308594]
-                ], 
-            'bboxes_labels': ['A green car'],
-            'polygons': [], 
-            'polygons_labels': []
-        }
-    }
-    """
-    assert text_input is not None, "Text input should not be None when calling open-vocabulary detection pipeline."
-    results = results[task_prompt]
-    # parse florence-2 detection results
-    input_boxes = np.array(results["bboxes"])
-    print(results)
-    class_names = results["bboxes_labels"]
-    class_ids = np.array(list(range(len(class_names))))
-    
-    # predict mask with SAM 2
-    sam2_predictor.set_image(np.array(image))
-    masks, scores, logits = sam2_predictor.predict(
-        point_coords=None,
-        point_labels=None,
-        box=input_boxes,
-        multimask_output=False,
-    )
-    
-    if masks.ndim == 4:
-        masks = masks.squeeze(1)
-    
-    # specify labels
-    labels = [
-        f"{class_name}" for class_name in class_names
-    ]
-    
-    # visualization results
-    img = cv2.imread(image_path)
-    detections = sv.Detections(
-        xyxy=input_boxes,
-        mask=masks.astype(bool),
-        class_id=class_ids
-    )
-    
-    box_annotator = sv.BoxAnnotator()
-    annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-    
-    label_annotator = sv.LabelAnnotator()
-    annotated_frame = label_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels)
-    cv2.imwrite(os.path.join(output_dir, "grounded_sam2_florence2_open_vocabulary_detection.jpg"), annotated_frame)
-    
-    mask_annotator = sv.MaskAnnotator()
-    annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-    cv2.imwrite(os.path.join(output_dir, "grounded_sam2_florence2_open_vocabulary_detection_with_mask.jpg"), annotated_frame)
-
-    print(f'Successfully save annotated image to "{output_dir}"')
-
-if __name__ == "__main__":
-
-    parser = argparse.ArgumentParser("Grounded SAM 2 Florence-2 Demos", add_help=True)
-    parser.add_argument("--image_path", type=str, default="./notebooks/images/cars.jpg", required=True, help="path to image file")
-    parser.add_argument("--pipeline", type=str, default="object_detection_segmentation", required=True, help="path to image file")
-    parser.add_argument("--text_input", type=str, default=None, required=False, help="path to image file")
-    args = parser.parse_args()
-
-    IMAGE_PATH = args.image_path
-    PIPELINE = args.pipeline
-    INPUT_TEXT = args.text_input
-
-    print(f"Running pipeline: {PIPELINE} now.")
-
-    if PIPELINE == "object_detection_segmentation":
-        # pipeline-1: detection + segmentation
-        object_detection_and_segmentation(
-            florence2_model=florence2_model,
-            florence2_processor=florence2_processor,
-            sam2_predictor=sam2_predictor,
-            image_path=IMAGE_PATH
-        )
-    elif PIPELINE == "dense_region_caption_segmentation":
-        # pipeline-2: dense region caption + segmentation
-        dense_region_caption_and_segmentation(
-            florence2_model=florence2_model,
-            florence2_processor=florence2_processor,
-            sam2_predictor=sam2_predictor,
-            image_path=IMAGE_PATH
-        )
-    elif PIPELINE == "region_proposal_segmentation":
-        # pipeline-3: dense region caption + segmentation
-        region_proposal_and_segmentation(
-            florence2_model=florence2_model,
-            florence2_processor=florence2_processor,
-            sam2_predictor=sam2_predictor,
-            image_path=IMAGE_PATH
-        )
-    elif PIPELINE == "phrase_grounding_segmentation":
-        # pipeline-4: phrase grounding + segmentation
-        phrase_grounding_and_segmentation(
-            florence2_model=florence2_model,
-            florence2_processor=florence2_processor,
-            sam2_predictor=sam2_predictor,
-            image_path=IMAGE_PATH,
-            text_input=INPUT_TEXT
-        )
-    elif PIPELINE == "referring_expression_segmentation":
-        # pipeline-5: referring segmentation + segmentation
-        referring_expression_segmentation(
-            florence2_model=florence2_model,
-            florence2_processor=florence2_processor,
-            sam2_predictor=sam2_predictor,
-            image_path=IMAGE_PATH,
-            text_input=INPUT_TEXT
-        )
-    elif PIPELINE == "open_vocabulary_detection_segmentation":
-        # pipeline-6: open-vocabulary detection + segmentation
-        open_vocabulary_detection_and_segmentation(
-            florence2_model=florence2_model,
-            florence2_processor=florence2_processor,
-            sam2_predictor=sam2_predictor,
-            image_path=IMAGE_PATH,
-            text_input=INPUT_TEXT
-        )
-    else:
-        raise NotImplementedError(f"Pipeline: {args.pipeline} is not implemented at this time")

grounded_sam2_gd1.5_demo.py

@@ -1,249 +0,0 @@
-# dds cloudapi for Grounding DINO 1.5
-from dds_cloudapi_sdk import Config
-from dds_cloudapi_sdk import Client
-from dds_cloudapi_sdk import DetectionTask
-from dds_cloudapi_sdk import TextPrompt
-from dds_cloudapi_sdk import DetectionModel
-from dds_cloudapi_sdk import DetectionTarget
-
-import os
-import cv2
-import json
-import torch
-import tempfile
-import numpy as np
-import supervision as sv
-import pycocotools.mask as mask_util
-from pathlib import Path
-from PIL import Image
-from sam2.build_sam import build_sam2
-from sam2.sam2_image_predictor import SAM2ImagePredictor
-
-"""
-Hyper parameters
-"""
-API_TOKEN = "Your API token"
-TEXT_PROMPT = "car . building ."
-IMG_PATH = "notebooks/images/cars.jpg"
-SAM2_CHECKPOINT = "./checkpoints/sam2.1_hiera_large.pt"
-SAM2_MODEL_CONFIG = "configs/sam2.1/sam2.1_hiera_l.yaml"
-GROUNDING_MODEL = DetectionModel.GDino1_5_Pro # DetectionModel.GDino1_6_Pro
-BOX_THRESHOLD = 0.2
-WITH_SLICE_INFERENCE = False
-SLICE_WH = (480, 480)
-OVERLAP_RATIO = (0.2, 0.2)
-DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
-OUTPUT_DIR = Path("outputs/grounded_sam2_gd1.5_demo")
-DUMP_JSON_RESULTS = True
-
-# create output directory
-OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
-
-"""
-Prompt Grounding DINO 1.5 with Text for Box Prompt Generation with Cloud API
-"""
-# Step 1: initialize the config
-token = API_TOKEN
-config = Config(token)
-
-# Step 2: initialize the client
-client = Client(config)
-
-# Step 3: run the task by DetectionTask class
-# image_url = "https://algosplt.oss-cn-shenzhen.aliyuncs.com/test_files/tasks/detection/iron_man.jpg"
-# if you are processing local image file, upload them to DDS server to get the image url
-
-classes = [x.strip().lower() for x in TEXT_PROMPT.split('.') if x]
-class_name_to_id = {name: id for id, name in enumerate(classes)}
-class_id_to_name = {id: name for name, id in class_name_to_id.items()}
-
-if WITH_SLICE_INFERENCE:
-    def callback(image_slice: np.ndarray) -> sv.Detections:
-        print("Inference on image slice")
-        # save the img as temp img file for GD-1.5 API usage
-        with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as tmpfile:
-            temp_filename = tmpfile.name
-        cv2.imwrite(temp_filename, image_slice)
-        image_url = client.upload_file(temp_filename)
-        task = DetectionTask(
-            image_url=image_url,
-            prompts=[TextPrompt(text=TEXT_PROMPT)],
-            targets=[DetectionTarget.BBox],  # detect bbox
-            model=GROUNDING_MODEL,  # detect with GroundingDino-1.5-Pro model
-            bbox_threshold=BOX_THRESHOLD, # box confidence threshold
-        )
-        client.run_task(task)
-        result = task.result
-        # detele the tempfile
-        os.remove(temp_filename)
-        
-        input_boxes = []
-        confidences = []
-        class_ids = []
-        objects = result.objects
-        for idx, obj in enumerate(objects):
-            input_boxes.append(obj.bbox)
-            confidences.append(obj.score)
-            cls_name = obj.category.lower().strip()
-            class_ids.append(class_name_to_id[cls_name])
-        # ensure input_boxes with shape (_, 4)
-        input_boxes = np.array(input_boxes).reshape(-1, 4)
-        class_ids = np.array(class_ids)
-        confidences = np.array(confidences)
-        return sv.Detections(xyxy=input_boxes, confidence=confidences, class_id=class_ids)
-    
-    slicer = sv.InferenceSlicer(
-        callback=callback,
-        slice_wh=SLICE_WH,
-        overlap_ratio_wh=OVERLAP_RATIO,
-        iou_threshold=0.5,
-        overlap_filter_strategy=sv.OverlapFilter.NON_MAX_SUPPRESSION
-        )
-    detections = slicer(cv2.imread(IMG_PATH))
-    class_names = [class_id_to_name[id] for id in detections.class_id]
-    confidences = detections.confidence
-    class_ids = detections.class_id
-    input_boxes = detections.xyxy
-else:
-    image_url = client.upload_file(IMG_PATH)
-
-    task = DetectionTask(
-        image_url=image_url,
-        prompts=[TextPrompt(text=TEXT_PROMPT)],
-        targets=[DetectionTarget.BBox],  # detect bbox
-        model=GROUNDING_MODEL,  # detect with GroundingDINO-1.5-Pro model
-        bbox_threshold=BOX_THRESHOLD, # box confidence threshold
-    )
-
-    client.run_task(task)
-    result = task.result
-
-    objects = result.objects  # the list of detected objects
-
-
-    input_boxes = []
-    confidences = []
-    class_names = []
-    class_ids = []
-
-    for idx, obj in enumerate(objects):
-        input_boxes.append(obj.bbox)
-        confidences.append(obj.score)
-        cls_name = obj.category.lower().strip()
-        class_names.append(cls_name)
-        class_ids.append(class_name_to_id[cls_name])
-
-    input_boxes = np.array(input_boxes)
-    class_ids = np.array(class_ids)
-
-"""
-Init SAM 2 Model and Predict Mask with Box Prompt
-"""
-
-# environment settings
-# use bfloat16
-torch.autocast(device_type=DEVICE, dtype=torch.bfloat16).__enter__()
-
-if torch.cuda.get_device_properties(0).major >= 8:
-    # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)
-    torch.backends.cuda.matmul.allow_tf32 = True
-    torch.backends.cudnn.allow_tf32 = True
-
-# build SAM2 image predictor
-sam2_checkpoint = SAM2_CHECKPOINT
-model_cfg = SAM2_MODEL_CONFIG
-sam2_model = build_sam2(model_cfg, sam2_checkpoint, device=DEVICE)
-sam2_predictor = SAM2ImagePredictor(sam2_model)
-
-image = Image.open(IMG_PATH)
-
-sam2_predictor.set_image(np.array(image.convert("RGB")))
-
-masks, scores, logits = sam2_predictor.predict(
-    point_coords=None,
-    point_labels=None,
-    box=input_boxes,
-    multimask_output=False,
-)
-
-
-"""
-Post-process the output of the model to get the masks, scores, and logits for visualization
-"""
-# convert the shape to (n, H, W)
-if masks.ndim == 4:
-    masks = masks.squeeze(1)
-
-
-"""
-Visualization the Predict Results
-"""
-
-labels = [
-    f"{class_name} {confidence:.2f}"
-    for class_name, confidence
-    in zip(class_names, confidences)
-]
-
-"""
-Visualize image with supervision useful API
-"""
-img = cv2.imread(IMG_PATH)
-detections = sv.Detections(
-    xyxy=input_boxes,  # (n, 4)
-    mask=masks.astype(bool),  # (n, h, w)
-    class_id=class_ids
-)
-
-box_annotator = sv.BoxAnnotator()
-annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-
-label_annotator = sv.LabelAnnotator()
-annotated_frame = label_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels)
-cv2.imwrite(os.path.join(OUTPUT_DIR, "groundingdino_annotated_image.jpg"), annotated_frame)
-
-mask_annotator = sv.MaskAnnotator()
-annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-cv2.imwrite(os.path.join(OUTPUT_DIR, "grounded_sam2_annotated_image_with_mask.jpg"), annotated_frame)
-
-print(f'Annotated image has already been saved as to "{OUTPUT_DIR}"')
-
-"""
-Dump the results in standard format and save as json files
-"""
-
-def single_mask_to_rle(mask):
-    rle = mask_util.encode(np.array(mask[:, :, None], order="F", dtype="uint8"))[0]
-    rle["counts"] = rle["counts"].decode("utf-8")
-    return rle
-
-if DUMP_JSON_RESULTS:
-    print("Start dumping the annotation...")
-    # convert mask into rle format
-    mask_rles = [single_mask_to_rle(mask) for mask in masks]
-
-    input_boxes = input_boxes.tolist()
-    scores = scores.tolist()
-    # FIXME: class_names should be a list of strings without spaces
-    class_names = [class_name.strip() for class_name in class_names]
-    # save the results in standard format
-    results = {
-        "image_path": IMG_PATH,
-        "annotations" : [
-            {
-                "class_name": class_name,
-                "bbox": box,
-                "segmentation": mask_rle,
-                "score": score,
-            }
-            for class_name, box, mask_rle, score in zip(class_names, input_boxes, mask_rles, scores)
-        ],
-        "box_format": "xyxy",
-        "img_width": image.width,
-        "img_height": image.height,
-    }
-    
-    with open(os.path.join(OUTPUT_DIR, "grounded_sam2_gd1.5_image_demo_results.json"), "w") as f:
-        json.dump(results, f, indent=4)
-
-    print(f'Annotation has already been saved to "{OUTPUT_DIR}"')

grounded_sam2_hf_model_demo.py

@@ -1,187 +0,0 @@
-import argparse
-import os
-import cv2
-import json
-import torch
-import numpy as np
-import supervision as sv
-import pycocotools.mask as mask_util
-from pathlib import Path
-from supervision.draw.color import ColorPalette
-from utils.supervision_utils import CUSTOM_COLOR_MAP
-from PIL import Image
-from sam2.build_sam import build_sam2
-from sam2.sam2_image_predictor import SAM2ImagePredictor
-from transformers import AutoProcessor, AutoModelForZeroShotObjectDetection 
-
-"""
-Hyper parameters
-"""
-parser = argparse.ArgumentParser()
-parser.add_argument('--grounding-model', default="IDEA-Research/grounding-dino-tiny")
-parser.add_argument("--text-prompt", default="car. tire.")
-parser.add_argument("--img-path", default="notebooks/images/truck.jpg")
-parser.add_argument("--sam2-checkpoint", default="./checkpoints/sam2.1_hiera_large.pt")
-parser.add_argument("--sam2-model-config", default="configs/sam2.1/sam2.1_hiera_l.yaml")
-parser.add_argument("--output-dir", default="outputs/test_sam2.1")
-parser.add_argument("--no-dump-json", action="store_true")
-parser.add_argument("--force-cpu", action="store_true")
-args = parser.parse_args()
-
-GROUNDING_MODEL = args.grounding_model
-TEXT_PROMPT = args.text_prompt
-IMG_PATH = args.img_path
-SAM2_CHECKPOINT = args.sam2_checkpoint
-SAM2_MODEL_CONFIG = args.sam2_model_config
-DEVICE = "cuda" if torch.cuda.is_available() and not args.force_cpu else "cpu"
-OUTPUT_DIR = Path(args.output_dir)
-DUMP_JSON_RESULTS = not args.no_dump_json
-
-# create output directory
-OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
-
-# environment settings
-# use bfloat16
-torch.autocast(device_type=DEVICE, dtype=torch.bfloat16).__enter__()
-
-if torch.cuda.get_device_properties(0).major >= 8:
-    # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)
-    torch.backends.cuda.matmul.allow_tf32 = True
-    torch.backends.cudnn.allow_tf32 = True
-
-# build SAM2 image predictor
-sam2_checkpoint = SAM2_CHECKPOINT
-model_cfg = SAM2_MODEL_CONFIG
-sam2_model = build_sam2(model_cfg, sam2_checkpoint, device=DEVICE)
-sam2_predictor = SAM2ImagePredictor(sam2_model)
-
-# build grounding dino from huggingface
-model_id = GROUNDING_MODEL
-processor = AutoProcessor.from_pretrained(model_id)
-grounding_model = AutoModelForZeroShotObjectDetection.from_pretrained(model_id).to(DEVICE)
-
-
-# setup the input image and text prompt for SAM 2 and Grounding DINO
-# VERY important: text queries need to be lowercased + end with a dot
-text = TEXT_PROMPT
-img_path = IMG_PATH
-
-image = Image.open(img_path)
-
-sam2_predictor.set_image(np.array(image.convert("RGB")))
-
-inputs = processor(images=image, text=text, return_tensors="pt").to(DEVICE)
-with torch.no_grad():
-    outputs = grounding_model(**inputs)
-
-results = processor.post_process_grounded_object_detection(
-    outputs,
-    inputs.input_ids,
-    box_threshold=0.4,
-    text_threshold=0.3,
-    target_sizes=[image.size[::-1]]
-)
-
-"""
-Results is a list of dict with the following structure:
-[
-    {
-        'scores': tensor([0.7969, 0.6469, 0.6002, 0.4220], device='cuda:0'), 
-        'labels': ['car', 'tire', 'tire', 'tire'], 
-        'boxes': tensor([[  89.3244,  278.6940, 1710.3505,  851.5143],
-                        [1392.4701,  554.4064, 1628.6133,  777.5872],
-                        [ 436.1182,  621.8940,  676.5255,  851.6897],
-                        [1236.0990,  688.3547, 1400.2427,  753.1256]], device='cuda:0')
-    }
-]
-"""
-
-# get the box prompt for SAM 2
-input_boxes = results[0]["boxes"].cpu().numpy()
-
-masks, scores, logits = sam2_predictor.predict(
-    point_coords=None,
-    point_labels=None,
-    box=input_boxes,
-    multimask_output=False,
-)
-
-
-"""
-Post-process the output of the model to get the masks, scores, and logits for visualization
-"""
-# convert the shape to (n, H, W)
-if masks.ndim == 4:
-    masks = masks.squeeze(1)
-
-
-confidences = results[0]["scores"].cpu().numpy().tolist()
-class_names = results[0]["labels"]
-class_ids = np.array(list(range(len(class_names))))
-
-labels = [
-    f"{class_name} {confidence:.2f}"
-    for class_name, confidence
-    in zip(class_names, confidences)
-]
-
-"""
-Visualize image with supervision useful API
-"""
-img = cv2.imread(img_path)
-detections = sv.Detections(
-    xyxy=input_boxes,  # (n, 4)
-    mask=masks.astype(bool),  # (n, h, w)
-    class_id=class_ids
-)
-
-"""
-Note that if you want to use default color map,
-you can set color=ColorPalette.DEFAULT
-"""
-box_annotator = sv.BoxAnnotator(color=ColorPalette.from_hex(CUSTOM_COLOR_MAP))
-annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-
-label_annotator = sv.LabelAnnotator(color=ColorPalette.from_hex(CUSTOM_COLOR_MAP))
-annotated_frame = label_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels)
-cv2.imwrite(os.path.join(OUTPUT_DIR, "groundingdino_annotated_image.jpg"), annotated_frame)
-
-mask_annotator = sv.MaskAnnotator(color=ColorPalette.from_hex(CUSTOM_COLOR_MAP))
-annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-cv2.imwrite(os.path.join(OUTPUT_DIR, "grounded_sam2_annotated_image_with_mask.jpg"), annotated_frame)
-
-
-"""
-Dump the results in standard format and save as json files
-"""
-
-def single_mask_to_rle(mask):
-    rle = mask_util.encode(np.array(mask[:, :, None], order="F", dtype="uint8"))[0]
-    rle["counts"] = rle["counts"].decode("utf-8")
-    return rle
-
-if DUMP_JSON_RESULTS:
-    # convert mask into rle format
-    mask_rles = [single_mask_to_rle(mask) for mask in masks]
-
-    input_boxes = input_boxes.tolist()
-    scores = scores.tolist()
-    # save the results in standard format
-    results = {
-        "image_path": img_path,
-        "annotations" : [
-            {
-                "class_name": class_name,
-                "bbox": box,
-                "segmentation": mask_rle,
-                "score": score,
-            }
-            for class_name, box, mask_rle, score in zip(class_names, input_boxes, mask_rles, scores)
-        ],
-        "box_format": "xyxy",
-        "img_width": image.width,
-        "img_height": image.height,
-    }
-    
-    with open(os.path.join(OUTPUT_DIR, "grounded_sam2_hf_model_demo_results.json"), "w") as f:
-        json.dump(results, f, indent=4)

grounded_sam2_local_demo.py

@@ -1,160 +0,0 @@
-import os
-import cv2
-import json
-import torch
-import numpy as np
-import supervision as sv
-import pycocotools.mask as mask_util
-from pathlib import Path
-from torchvision.ops import box_convert
-from sam2.build_sam import build_sam2
-from sam2.sam2_image_predictor import SAM2ImagePredictor
-from grounding_dino.groundingdino.util.inference import load_model, load_image, predict
-
-"""
-Hyper parameters
-"""
-TEXT_PROMPT = "car. tire."
-IMG_PATH = "notebooks/images/truck.jpg"
-SAM2_CHECKPOINT = "./checkpoints/sam2.1_hiera_large.pt"
-SAM2_MODEL_CONFIG = "configs/sam2.1/sam2.1_hiera_l.yaml"
-GROUNDING_DINO_CONFIG = "grounding_dino/groundingdino/config/GroundingDINO_SwinT_OGC.py"
-GROUNDING_DINO_CHECKPOINT = "gdino_checkpoints/groundingdino_swint_ogc.pth"
-BOX_THRESHOLD = 0.35
-TEXT_THRESHOLD = 0.25
-DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
-OUTPUT_DIR = Path("outputs/grounded_sam2_local_demo")
-DUMP_JSON_RESULTS = True
-
-# create output directory
-OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
-
-# environment settings
-# use bfloat16
-
-# build SAM2 image predictor
-sam2_checkpoint = SAM2_CHECKPOINT
-model_cfg = SAM2_MODEL_CONFIG
-sam2_model = build_sam2(model_cfg, sam2_checkpoint, device=DEVICE)
-sam2_predictor = SAM2ImagePredictor(sam2_model)
-
-# build grounding dino model
-grounding_model = load_model(
-    model_config_path=GROUNDING_DINO_CONFIG, 
-    model_checkpoint_path=GROUNDING_DINO_CHECKPOINT,
-    device=DEVICE
-)
-
-
-# setup the input image and text prompt for SAM 2 and Grounding DINO
-# VERY important: text queries need to be lowercased + end with a dot
-text = TEXT_PROMPT
-img_path = IMG_PATH
-
-image_source, image = load_image(img_path)
-
-sam2_predictor.set_image(image_source)
-
-boxes, confidences, labels = predict(
-    model=grounding_model,
-    image=image,
-    caption=text,
-    box_threshold=BOX_THRESHOLD,
-    text_threshold=TEXT_THRESHOLD,
-)
-
-# process the box prompt for SAM 2
-h, w, _ = image_source.shape
-boxes = boxes * torch.Tensor([w, h, w, h])
-input_boxes = box_convert(boxes=boxes, in_fmt="cxcywh", out_fmt="xyxy").numpy()
-
-
-# FIXME: figure how does this influence the G-DINO model
-torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__()
-
-if torch.cuda.get_device_properties(0).major >= 8:
-    # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)
-    torch.backends.cuda.matmul.allow_tf32 = True
-    torch.backends.cudnn.allow_tf32 = True
-
-masks, scores, logits = sam2_predictor.predict(
-    point_coords=None,
-    point_labels=None,
-    box=input_boxes,
-    multimask_output=False,
-)
-
-"""
-Post-process the output of the model to get the masks, scores, and logits for visualization
-"""
-# convert the shape to (n, H, W)
-if masks.ndim == 4:
-    masks = masks.squeeze(1)
-
-
-confidences = confidences.numpy().tolist()
-class_names = labels
-
-class_ids = np.array(list(range(len(class_names))))
-
-labels = [
-    f"{class_name} {confidence:.2f}"
-    for class_name, confidence
-    in zip(class_names, confidences)
-]
-
-"""
-Visualize image with supervision useful API
-"""
-img = cv2.imread(img_path)
-detections = sv.Detections(
-    xyxy=input_boxes,  # (n, 4)
-    mask=masks.astype(bool),  # (n, h, w)
-    class_id=class_ids
-)
-
-box_annotator = sv.BoxAnnotator()
-annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-
-label_annotator = sv.LabelAnnotator()
-annotated_frame = label_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels)
-cv2.imwrite(os.path.join(OUTPUT_DIR, "groundingdino_annotated_image.jpg"), annotated_frame)
-
-mask_annotator = sv.MaskAnnotator()
-annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-cv2.imwrite(os.path.join(OUTPUT_DIR, "grounded_sam2_annotated_image_with_mask.jpg"), annotated_frame)
-
-"""
-Dump the results in standard format and save as json files
-"""
-
-def single_mask_to_rle(mask):
-    rle = mask_util.encode(np.array(mask[:, :, None], order="F", dtype="uint8"))[0]
-    rle["counts"] = rle["counts"].decode("utf-8")
-    return rle
-
-if DUMP_JSON_RESULTS:
-    # convert mask into rle format
-    mask_rles = [single_mask_to_rle(mask) for mask in masks]
-
-    input_boxes = input_boxes.tolist()
-    scores = scores.tolist()
-    # save the results in standard format
-    results = {
-        "image_path": img_path,
-        "annotations" : [
-            {
-                "class_name": class_name,
-                "bbox": box,
-                "segmentation": mask_rle,
-                "score": score,
-            }
-            for class_name, box, mask_rle, score in zip(class_names, input_boxes, mask_rles, scores)
-        ],
-        "box_format": "xyxy",
-        "img_width": w,
-        "img_height": h,
-    }
-    
-    with open(os.path.join(OUTPUT_DIR, "grounded_sam2_local_image_demo_results.json"), "w") as f:
-        json.dump(results, f, indent=4)

grounded_sam2_tracking_demo.py

@@ -1,198 +0,0 @@
-import os
-import cv2
-import torch
-import numpy as np
-import supervision as sv
-from PIL import Image
-from sam2.build_sam import build_sam2_video_predictor, build_sam2
-from sam2.sam2_image_predictor import SAM2ImagePredictor
-from transformers import AutoProcessor, AutoModelForZeroShotObjectDetection 
-from utils.track_utils import sample_points_from_masks
-from utils.video_utils import create_video_from_images
-
-
-"""
-Step 1: Environment settings and model initialization
-"""
-# use bfloat16 for the entire notebook
-torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__()
-
-if torch.cuda.get_device_properties(0).major >= 8:
-    # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)
-    torch.backends.cuda.matmul.allow_tf32 = True
-    torch.backends.cudnn.allow_tf32 = True
-
-# init sam image predictor and video predictor model
-sam2_checkpoint = "./checkpoints/sam2.1_hiera_large.pt"
-model_cfg = "configs/sam2.1/sam2.1_hiera_l.yaml"
-
-video_predictor = build_sam2_video_predictor(model_cfg, sam2_checkpoint)
-sam2_image_model = build_sam2(model_cfg, sam2_checkpoint)
-image_predictor = SAM2ImagePredictor(sam2_image_model)
-
-
-# init grounding dino model from huggingface
-model_id = "IDEA-Research/grounding-dino-tiny"
-device = "cuda" if torch.cuda.is_available() else "cpu"
-processor = AutoProcessor.from_pretrained(model_id)
-grounding_model = AutoModelForZeroShotObjectDetection.from_pretrained(model_id).to(device)
-
-
-# setup the input image and text prompt for SAM 2 and Grounding DINO
-# VERY important: text queries need to be lowercased + end with a dot
-text = "car."
-
-# `video_dir` a directory of JPEG frames with filenames like `<frame_index>.jpg`  
-
-video_dir = "notebooks/videos/car"
-
-# scan all the JPEG frame names in this directory
-frame_names = [
-    p for p in os.listdir(video_dir)
-    if os.path.splitext(p)[-1] in [".jpg", ".jpeg", ".JPG", ".JPEG"]
-]
-frame_names.sort(key=lambda p: int(os.path.splitext(p)[0]))
-
-# init video predictor state
-inference_state = video_predictor.init_state(video_path=video_dir)
-
-ann_frame_idx = 0  # the frame index we interact with
-ann_obj_id = 1  # give a unique id to each object we interact with (it can be any integers)
-
-
-"""
-Step 2: Prompt Grounding DINO and SAM image predictor to get the box and mask for specific frame
-"""
-
-# prompt grounding dino to get the box coordinates on specific frame
-img_path = os.path.join(video_dir, frame_names[ann_frame_idx])
-image = Image.open(img_path)
-
-# run Grounding DINO on the image
-inputs = processor(images=image, text=text, return_tensors="pt").to(device)
-with torch.no_grad():
-    outputs = grounding_model(**inputs)
-
-results = processor.post_process_grounded_object_detection(
-    outputs,
-    inputs.input_ids,
-    box_threshold=0.25,
-    text_threshold=0.3,
-    target_sizes=[image.size[::-1]]
-)
-
-# prompt SAM image predictor to get the mask for the object
-image_predictor.set_image(np.array(image.convert("RGB")))
-
-# process the detection results
-input_boxes = results[0]["boxes"].cpu().numpy()
-OBJECTS = results[0]["labels"]
-
-# prompt SAM 2 image predictor to get the mask for the object
-masks, scores, logits = image_predictor.predict(
-    point_coords=None,
-    point_labels=None,
-    box=input_boxes,
-    multimask_output=False,
-)
-
-# convert the mask shape to (n, H, W)
-if masks.ndim == 3:
-    masks = masks[None]
-    scores = scores[None]
-    logits = logits[None]
-elif masks.ndim == 4:
-    masks = masks.squeeze(1)
-
-"""
-Step 3: Register each object's positive points to video predictor with seperate add_new_points call
-"""
-
-PROMPT_TYPE_FOR_VIDEO = "box" # or "point"
-
-assert PROMPT_TYPE_FOR_VIDEO in ["point", "box", "mask"], "SAM 2 video predictor only support point/box/mask prompt"
-
-# If you are using point prompts, we uniformly sample positive points based on the mask
-if PROMPT_TYPE_FOR_VIDEO == "point":
-    # sample the positive points from mask for each objects
-    all_sample_points = sample_points_from_masks(masks=masks, num_points=10)
-
-    for object_id, (label, points) in enumerate(zip(OBJECTS, all_sample_points), start=1):
-        labels = np.ones((points.shape[0]), dtype=np.int32)
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_points_or_box(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            points=points,
-            labels=labels,
-        )
-# Using box prompt
-elif PROMPT_TYPE_FOR_VIDEO == "box":
-    for object_id, (label, box) in enumerate(zip(OBJECTS, input_boxes), start=1):
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_points_or_box(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            box=box,
-        )
-# Using mask prompt is a more straightforward way
-elif PROMPT_TYPE_FOR_VIDEO == "mask":
-    for object_id, (label, mask) in enumerate(zip(OBJECTS, masks), start=1):
-        labels = np.ones((1), dtype=np.int32)
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_mask(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            mask=mask
-        )
-else:
-    raise NotImplementedError("SAM 2 video predictor only support point/box/mask prompts")
-
-
-"""
-Step 4: Propagate the video predictor to get the segmentation results for each frame
-"""
-video_segments = {}  # video_segments contains the per-frame segmentation results
-for out_frame_idx, out_obj_ids, out_mask_logits in video_predictor.propagate_in_video(inference_state):
-    video_segments[out_frame_idx] = {
-        out_obj_id: (out_mask_logits[i] > 0.0).cpu().numpy()
-        for i, out_obj_id in enumerate(out_obj_ids)
-    }
-
-"""
-Step 5: Visualize the segment results across the video and save them
-"""
-
-save_dir = "./tracking_results"
-
-if not os.path.exists(save_dir):
-    os.makedirs(save_dir)
-
-ID_TO_OBJECTS = {i: obj for i, obj in enumerate(OBJECTS, start=1)}
-for frame_idx, segments in video_segments.items():
-    img = cv2.imread(os.path.join(video_dir, frame_names[frame_idx]))
-    
-    object_ids = list(segments.keys())
-    masks = list(segments.values())
-    masks = np.concatenate(masks, axis=0)
-    
-    detections = sv.Detections(
-        xyxy=sv.mask_to_xyxy(masks),  # (n, 4)
-        mask=masks, # (n, h, w)
-        class_id=np.array(object_ids, dtype=np.int32),
-    )
-    box_annotator = sv.BoxAnnotator()
-    annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-    label_annotator = sv.LabelAnnotator()
-    annotated_frame = label_annotator.annotate(annotated_frame, detections=detections, labels=[ID_TO_OBJECTS[i] for i in object_ids])
-    mask_annotator = sv.MaskAnnotator()
-    annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-    cv2.imwrite(os.path.join(save_dir, f"annotated_frame_{frame_idx:05d}.jpg"), annotated_frame)
-
-
-"""
-Step 6: Convert the annotated frames to video
-"""
-
-output_video_path = "./children_tracking_demo_video.mp4"
-create_video_from_images(save_dir, output_video_path)

grounded_sam2_tracking_demo_custom_video_input_gd1.0_hf_model.py

@@ -1,214 +0,0 @@
-import os
-import cv2
-import torch
-import numpy as np
-import supervision as sv
-
-from pathlib import Path
-from tqdm import tqdm
-from PIL import Image
-from sam2.build_sam import build_sam2_video_predictor, build_sam2
-from sam2.sam2_image_predictor import SAM2ImagePredictor 
-from transformers import AutoProcessor, AutoModelForZeroShotObjectDetection
-from utils.track_utils import sample_points_from_masks
-from utils.video_utils import create_video_from_images
-
-"""
-Hyperparam for Ground and Tracking
-"""
-MODEL_ID = "IDEA-Research/grounding-dino-tiny"
-VIDEO_PATH = "./assets/hippopotamus.mp4"
-TEXT_PROMPT = "hippopotamus."
-OUTPUT_VIDEO_PATH = "./hippopotamus_tracking_demo.mp4"
-SOURCE_VIDEO_FRAME_DIR = "./custom_video_frames"
-SAVE_TRACKING_RESULTS_DIR = "./tracking_results"
-PROMPT_TYPE_FOR_VIDEO = "box" # choose from ["point", "box", "mask"]
-
-"""
-Step 1: Environment settings and model initialization for SAM 2
-"""
-# use bfloat16 for the entire notebook
-torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__()
-
-if torch.cuda.get_device_properties(0).major >= 8:
-    # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)
-    torch.backends.cuda.matmul.allow_tf32 = True
-    torch.backends.cudnn.allow_tf32 = True
-
-# init sam image predictor and video predictor model
-sam2_checkpoint = "./checkpoints/sam2.1_hiera_large.pt"
-model_cfg = "configs/sam2.1/sam2.1_hiera_l.yaml"
-
-video_predictor = build_sam2_video_predictor(model_cfg, sam2_checkpoint)
-sam2_image_model = build_sam2(model_cfg, sam2_checkpoint)
-image_predictor = SAM2ImagePredictor(sam2_image_model)
-
-# build grounding dino from huggingface
-model_id = MODEL_ID
-device = "cuda" if torch.cuda.is_available() else "cpu"
-processor = AutoProcessor.from_pretrained(model_id)
-grounding_model = AutoModelForZeroShotObjectDetection.from_pretrained(model_id).to(device)
-
-
-"""
-Custom video input directly using video files
-"""
-video_info = sv.VideoInfo.from_video_path(VIDEO_PATH)  # get video info
-print(video_info)
-frame_generator = sv.get_video_frames_generator(VIDEO_PATH, stride=1, start=0, end=None)
-
-# saving video to frames
-source_frames = Path(SOURCE_VIDEO_FRAME_DIR)
-source_frames.mkdir(parents=True, exist_ok=True)
-
-with sv.ImageSink(
-    target_dir_path=source_frames, 
-    overwrite=True, 
-    image_name_pattern="{:05d}.jpg"
-) as sink:
-    for frame in tqdm(frame_generator, desc="Saving Video Frames"):
-        sink.save_image(frame)
-
-# scan all the JPEG frame names in this directory
-frame_names = [
-    p for p in os.listdir(SOURCE_VIDEO_FRAME_DIR)
-    if os.path.splitext(p)[-1] in [".jpg", ".jpeg", ".JPG", ".JPEG"]
-]
-frame_names.sort(key=lambda p: int(os.path.splitext(p)[0]))
-
-# init video predictor state
-inference_state = video_predictor.init_state(video_path=SOURCE_VIDEO_FRAME_DIR)
-
-ann_frame_idx = 0  # the frame index we interact with
-"""
-Step 2: Prompt Grounding DINO 1.5 with Cloud API for box coordinates
-"""
-
-# prompt grounding dino to get the box coordinates on specific frame
-img_path = os.path.join(SOURCE_VIDEO_FRAME_DIR, frame_names[ann_frame_idx])
-image = Image.open(img_path)
-inputs = processor(images=image, text=TEXT_PROMPT, return_tensors="pt").to(device)
-with torch.no_grad():
-    outputs = grounding_model(**inputs)
-
-results = processor.post_process_grounded_object_detection(
-    outputs,
-    inputs.input_ids,
-    box_threshold=0.4,
-    text_threshold=0.3,
-    target_sizes=[image.size[::-1]]
-)
-
-input_boxes = results[0]["boxes"].cpu().numpy()
-confidences = results[0]["scores"].cpu().numpy().tolist()
-class_names = results[0]["labels"]
-
-print(input_boxes)
-
-# prompt SAM image predictor to get the mask for the object
-image_predictor.set_image(np.array(image.convert("RGB")))
-
-# process the detection results
-OBJECTS = class_names
-
-print(OBJECTS)
-
-# prompt SAM 2 image predictor to get the mask for the object
-masks, scores, logits = image_predictor.predict(
-    point_coords=None,
-    point_labels=None,
-    box=input_boxes,
-    multimask_output=False,
-)
-# convert the mask shape to (n, H, W)
-if masks.ndim == 4:
-    masks = masks.squeeze(1)
-
-"""
-Step 3: Register each object's positive points to video predictor with seperate add_new_points call
-"""
-
-assert PROMPT_TYPE_FOR_VIDEO in ["point", "box", "mask"], "SAM 2 video predictor only support point/box/mask prompt"
-
-# If you are using point prompts, we uniformly sample positive points based on the mask
-if PROMPT_TYPE_FOR_VIDEO == "point":
-    # sample the positive points from mask for each objects
-    all_sample_points = sample_points_from_masks(masks=masks, num_points=10)
-
-    for object_id, (label, points) in enumerate(zip(OBJECTS, all_sample_points), start=1):
-        labels = np.ones((points.shape[0]), dtype=np.int32)
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_points_or_box(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            points=points,
-            labels=labels,
-        )
-# Using box prompt
-elif PROMPT_TYPE_FOR_VIDEO == "box":
-    for object_id, (label, box) in enumerate(zip(OBJECTS, input_boxes), start=1):
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_points_or_box(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            box=box,
-        )
-# Using mask prompt is a more straightforward way
-elif PROMPT_TYPE_FOR_VIDEO == "mask":
-    for object_id, (label, mask) in enumerate(zip(OBJECTS, masks), start=1):
-        labels = np.ones((1), dtype=np.int32)
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_mask(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            mask=mask
-        )
-else:
-    raise NotImplementedError("SAM 2 video predictor only support point/box/mask prompts")
-
-
-"""
-Step 4: Propagate the video predictor to get the segmentation results for each frame
-"""
-video_segments = {}  # video_segments contains the per-frame segmentation results
-for out_frame_idx, out_obj_ids, out_mask_logits in video_predictor.propagate_in_video(inference_state):
-    video_segments[out_frame_idx] = {
-        out_obj_id: (out_mask_logits[i] > 0.0).cpu().numpy()
-        for i, out_obj_id in enumerate(out_obj_ids)
-    }
-
-"""
-Step 5: Visualize the segment results across the video and save them
-"""
-
-if not os.path.exists(SAVE_TRACKING_RESULTS_DIR):
-    os.makedirs(SAVE_TRACKING_RESULTS_DIR)
-
-ID_TO_OBJECTS = {i: obj for i, obj in enumerate(OBJECTS, start=1)}
-
-for frame_idx, segments in video_segments.items():
-    img = cv2.imread(os.path.join(SOURCE_VIDEO_FRAME_DIR, frame_names[frame_idx]))
-    
-    object_ids = list(segments.keys())
-    masks = list(segments.values())
-    masks = np.concatenate(masks, axis=0)
-    
-    detections = sv.Detections(
-        xyxy=sv.mask_to_xyxy(masks),  # (n, 4)
-        mask=masks, # (n, h, w)
-        class_id=np.array(object_ids, dtype=np.int32),
-    )
-    box_annotator = sv.BoxAnnotator()
-    annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-    label_annotator = sv.LabelAnnotator()
-    annotated_frame = label_annotator.annotate(annotated_frame, detections=detections, labels=[ID_TO_OBJECTS[i] for i in object_ids])
-    mask_annotator = sv.MaskAnnotator()
-    annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-    cv2.imwrite(os.path.join(SAVE_TRACKING_RESULTS_DIR, f"annotated_frame_{frame_idx:05d}.jpg"), annotated_frame)
-
-
-"""
-Step 6: Convert the annotated frames to video
-"""
-
-create_video_from_images(SAVE_TRACKING_RESULTS_DIR, OUTPUT_VIDEO_PATH)

grounded_sam2_tracking_demo_custom_video_input_gd1.0_local_model.py

@@ -1,220 +0,0 @@
-import os
-import cv2
-import torch
-import numpy as np
-import supervision as sv
-from torchvision.ops import box_convert
-from pathlib import Path
-from tqdm import tqdm
-from PIL import Image
-from sam2.build_sam import build_sam2_video_predictor, build_sam2
-from sam2.sam2_image_predictor import SAM2ImagePredictor 
-from grounding_dino.groundingdino.util.inference import load_model, load_image, predict
-from utils.track_utils import sample_points_from_masks
-from utils.video_utils import create_video_from_images
-
-"""
-Hyperparam for Ground and Tracking
-"""
-GROUNDING_DINO_CONFIG = "grounding_dino/groundingdino/config/GroundingDINO_SwinT_OGC.py"
-GROUNDING_DINO_CHECKPOINT = "gdino_checkpoints/groundingdino_swint_ogc.pth"
-BOX_THRESHOLD = 0.35
-TEXT_THRESHOLD = 0.25
-VIDEO_PATH = "./assets/hippopotamus.mp4"
-TEXT_PROMPT = "hippopotamus."
-OUTPUT_VIDEO_PATH = "./hippopotamus_tracking_demo.mp4"
-SOURCE_VIDEO_FRAME_DIR = "./custom_video_frames"
-SAVE_TRACKING_RESULTS_DIR = "./tracking_results"
-PROMPT_TYPE_FOR_VIDEO = "box" # choose from ["point", "box", "mask"]
-DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
-
-"""
-Step 1: Environment settings and model initialization for Grounding DINO and SAM 2
-"""
-# build grounding dino model from local path
-grounding_model = load_model(
-    model_config_path=GROUNDING_DINO_CONFIG, 
-    model_checkpoint_path=GROUNDING_DINO_CHECKPOINT,
-    device=DEVICE
-)
-
-
-# init sam image predictor and video predictor model
-sam2_checkpoint = "./checkpoints/sam2.1_hiera_large.pt"
-model_cfg = "configs/sam2.1/sam2.1_hiera_l.yaml"
-
-video_predictor = build_sam2_video_predictor(model_cfg, sam2_checkpoint)
-sam2_image_model = build_sam2(model_cfg, sam2_checkpoint)
-image_predictor = SAM2ImagePredictor(sam2_image_model)
-
-
-"""
-Custom video input directly using video files
-"""
-video_info = sv.VideoInfo.from_video_path(VIDEO_PATH)  # get video info
-print(video_info)
-frame_generator = sv.get_video_frames_generator(VIDEO_PATH, stride=1, start=0, end=None)
-
-# saving video to frames
-source_frames = Path(SOURCE_VIDEO_FRAME_DIR)
-source_frames.mkdir(parents=True, exist_ok=True)
-
-with sv.ImageSink(
-    target_dir_path=source_frames, 
-    overwrite=True, 
-    image_name_pattern="{:05d}.jpg"
-) as sink:
-    for frame in tqdm(frame_generator, desc="Saving Video Frames"):
-        sink.save_image(frame)
-
-# scan all the JPEG frame names in this directory
-frame_names = [
-    p for p in os.listdir(SOURCE_VIDEO_FRAME_DIR)
-    if os.path.splitext(p)[-1] in [".jpg", ".jpeg", ".JPG", ".JPEG"]
-]
-frame_names.sort(key=lambda p: int(os.path.splitext(p)[0]))
-
-# init video predictor state
-inference_state = video_predictor.init_state(video_path=SOURCE_VIDEO_FRAME_DIR)
-
-ann_frame_idx = 0  # the frame index we interact with
-"""
-Step 2: Prompt Grounding DINO 1.5 with Cloud API for box coordinates
-"""
-
-# prompt grounding dino to get the box coordinates on specific frame
-img_path = os.path.join(SOURCE_VIDEO_FRAME_DIR, frame_names[ann_frame_idx])
-image_source, image = load_image(img_path)
-
-boxes, confidences, labels = predict(
-    model=grounding_model,
-    image=image,
-    caption=TEXT_PROMPT,
-    box_threshold=BOX_THRESHOLD,
-    text_threshold=TEXT_THRESHOLD,
-)
-
-# process the box prompt for SAM 2
-h, w, _ = image_source.shape
-boxes = boxes * torch.Tensor([w, h, w, h])
-input_boxes = box_convert(boxes=boxes, in_fmt="cxcywh", out_fmt="xyxy").numpy()
-confidences = confidences.numpy().tolist()
-class_names = labels
-
-print(input_boxes)
-
-# prompt SAM image predictor to get the mask for the object
-image_predictor.set_image(image_source)
-
-# process the detection results
-OBJECTS = class_names
-
-print(OBJECTS)
-
-# FIXME: figure how does this influence the G-DINO model
-torch.autocast(device_type=DEVICE, dtype=torch.bfloat16).__enter__()
-
-if torch.cuda.get_device_properties(0).major >= 8:
-    # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)
-    torch.backends.cuda.matmul.allow_tf32 = True
-    torch.backends.cudnn.allow_tf32 = True
-
-# prompt SAM 2 image predictor to get the mask for the object
-masks, scores, logits = image_predictor.predict(
-    point_coords=None,
-    point_labels=None,
-    box=input_boxes,
-    multimask_output=False,
-)
-# convert the mask shape to (n, H, W)
-if masks.ndim == 4:
-    masks = masks.squeeze(1)
-
-"""
-Step 3: Register each object's positive points to video predictor with seperate add_new_points call
-"""
-
-assert PROMPT_TYPE_FOR_VIDEO in ["point", "box", "mask"], "SAM 2 video predictor only support point/box/mask prompt"
-
-# If you are using point prompts, we uniformly sample positive points based on the mask
-if PROMPT_TYPE_FOR_VIDEO == "point":
-    # sample the positive points from mask for each objects
-    all_sample_points = sample_points_from_masks(masks=masks, num_points=10)
-
-    for object_id, (label, points) in enumerate(zip(OBJECTS, all_sample_points), start=1):
-        labels = np.ones((points.shape[0]), dtype=np.int32)
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_points_or_box(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            points=points,
-            labels=labels,
-        )
-# Using box prompt
-elif PROMPT_TYPE_FOR_VIDEO == "box":
-    for object_id, (label, box) in enumerate(zip(OBJECTS, input_boxes), start=1):
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_points_or_box(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            box=box,
-        )
-# Using mask prompt is a more straightforward way
-elif PROMPT_TYPE_FOR_VIDEO == "mask":
-    for object_id, (label, mask) in enumerate(zip(OBJECTS, masks), start=1):
-        labels = np.ones((1), dtype=np.int32)
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_mask(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            mask=mask
-        )
-else:
-    raise NotImplementedError("SAM 2 video predictor only support point/box/mask prompts")
-
-
-"""
-Step 4: Propagate the video predictor to get the segmentation results for each frame
-"""
-video_segments = {}  # video_segments contains the per-frame segmentation results
-for out_frame_idx, out_obj_ids, out_mask_logits in video_predictor.propagate_in_video(inference_state):
-    video_segments[out_frame_idx] = {
-        out_obj_id: (out_mask_logits[i] > 0.0).cpu().numpy()
-        for i, out_obj_id in enumerate(out_obj_ids)
-    }
-
-"""
-Step 5: Visualize the segment results across the video and save them
-"""
-
-if not os.path.exists(SAVE_TRACKING_RESULTS_DIR):
-    os.makedirs(SAVE_TRACKING_RESULTS_DIR)
-
-ID_TO_OBJECTS = {i: obj for i, obj in enumerate(OBJECTS, start=1)}
-
-for frame_idx, segments in video_segments.items():
-    img = cv2.imread(os.path.join(SOURCE_VIDEO_FRAME_DIR, frame_names[frame_idx]))
-    
-    object_ids = list(segments.keys())
-    masks = list(segments.values())
-    masks = np.concatenate(masks, axis=0)
-    
-    detections = sv.Detections(
-        xyxy=sv.mask_to_xyxy(masks),  # (n, 4)
-        mask=masks, # (n, h, w)
-        class_id=np.array(object_ids, dtype=np.int32),
-    )
-    box_annotator = sv.BoxAnnotator()
-    annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-    label_annotator = sv.LabelAnnotator()
-    annotated_frame = label_annotator.annotate(annotated_frame, detections=detections, labels=[ID_TO_OBJECTS[i] for i in object_ids])
-    mask_annotator = sv.MaskAnnotator()
-    annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-    cv2.imwrite(os.path.join(SAVE_TRACKING_RESULTS_DIR, f"annotated_frame_{frame_idx:05d}.jpg"), annotated_frame)
-
-
-"""
-Step 6: Convert the annotated frames to video
-"""
-
-create_video_from_images(SAVE_TRACKING_RESULTS_DIR, OUTPUT_VIDEO_PATH)

grounded_sam2_tracking_demo_custom_video_input_gd1.5.py

@@ -1,239 +0,0 @@
-# dds cloudapi for Grounding DINO 1.5
-from dds_cloudapi_sdk import Config
-from dds_cloudapi_sdk import Client
-from dds_cloudapi_sdk import DetectionTask
-from dds_cloudapi_sdk import TextPrompt
-from dds_cloudapi_sdk import DetectionModel
-from dds_cloudapi_sdk import DetectionTarget
-
-import os
-import cv2
-import torch
-import numpy as np
-import supervision as sv
-
-from pathlib import Path
-from tqdm import tqdm
-from PIL import Image
-from sam2.build_sam import build_sam2_video_predictor, build_sam2
-from sam2.sam2_image_predictor import SAM2ImagePredictor 
-from utils.track_utils import sample_points_from_masks
-from utils.video_utils import create_video_from_images
-
-"""
-Hyperparam for Ground and Tracking
-"""
-VIDEO_PATH = "./assets/hippopotamus.mp4"
-TEXT_PROMPT = "hippopotamus."
-OUTPUT_VIDEO_PATH = "./hippopotamus_tracking_demo.mp4"
-SOURCE_VIDEO_FRAME_DIR = "./custom_video_frames"
-SAVE_TRACKING_RESULTS_DIR = "./tracking_results"
-API_TOKEN_FOR_GD1_5 = "Your API token"
-PROMPT_TYPE_FOR_VIDEO = "box" # choose from ["point", "box", "mask"]
-BOX_THRESHOLD = 0.2
-
-"""
-Step 1: Environment settings and model initialization for SAM 2
-"""
-# use bfloat16 for the entire notebook
-torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__()
-
-if torch.cuda.get_device_properties(0).major >= 8:
-    # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)
-    torch.backends.cuda.matmul.allow_tf32 = True
-    torch.backends.cudnn.allow_tf32 = True
-
-# init sam image predictor and video predictor model
-sam2_checkpoint = "./checkpoints/sam2.1_hiera_large.pt"
-model_cfg = "configs/sam2.1/sam2.1_hiera_l.yaml"
-
-video_predictor = build_sam2_video_predictor(model_cfg, sam2_checkpoint)
-sam2_image_model = build_sam2(model_cfg, sam2_checkpoint)
-image_predictor = SAM2ImagePredictor(sam2_image_model)
-
-
-# # `video_dir` a directory of JPEG frames with filenames like `<frame_index>.jpg`
-# video_dir = "notebooks/videos/bedroom"
-
-"""
-Custom video input directly using video files
-"""
-video_info = sv.VideoInfo.from_video_path(VIDEO_PATH)  # get video info
-print(video_info)
-frame_generator = sv.get_video_frames_generator(VIDEO_PATH, stride=1, start=0, end=None)
-
-# saving video to frames
-source_frames = Path(SOURCE_VIDEO_FRAME_DIR)
-source_frames.mkdir(parents=True, exist_ok=True)
-
-with sv.ImageSink(
-    target_dir_path=source_frames, 
-    overwrite=True, 
-    image_name_pattern="{:05d}.jpg"
-) as sink:
-    for frame in tqdm(frame_generator, desc="Saving Video Frames"):
-        sink.save_image(frame)
-
-# scan all the JPEG frame names in this directory
-frame_names = [
-    p for p in os.listdir(SOURCE_VIDEO_FRAME_DIR)
-    if os.path.splitext(p)[-1] in [".jpg", ".jpeg", ".JPG", ".JPEG"]
-]
-frame_names.sort(key=lambda p: int(os.path.splitext(p)[0]))
-
-# init video predictor state
-inference_state = video_predictor.init_state(video_path=SOURCE_VIDEO_FRAME_DIR)
-
-ann_frame_idx = 0  # the frame index we interact with
-"""
-Step 2: Prompt Grounding DINO 1.5 with Cloud API for box coordinates
-"""
-
-# prompt grounding dino to get the box coordinates on specific frame
-img_path = os.path.join(SOURCE_VIDEO_FRAME_DIR, frame_names[ann_frame_idx])
-image = Image.open(img_path)
-
-# Step 1: initialize the config
-config = Config(API_TOKEN_FOR_GD1_5)
-
-# Step 2: initialize the client
-client = Client(config)
-
-# Step 3: run the task by DetectionTask class
-# image_url = "https://algosplt.oss-cn-shenzhen.aliyuncs.com/test_files/tasks/detection/iron_man.jpg"
-# if you are processing local image file, upload them to DDS server to get the image url
-image_url = client.upload_file(img_path)
-
-task = DetectionTask(
-    image_url=image_url,
-    prompts=[TextPrompt(text=TEXT_PROMPT)],
-    targets=[DetectionTarget.BBox],  # detect bbox
-    model=DetectionModel.GDino1_6_Pro,  # detect with GroundingDino-1.5-Pro model
-    bbox_threshold=BOX_THRESHOLD,
-)
-
-client.run_task(task)
-result = task.result
-
-objects = result.objects  # the list of detected objects
-
-
-input_boxes = []
-confidences = []
-class_names = []
-
-for idx, obj in enumerate(objects):
-    input_boxes.append(obj.bbox)
-    confidences.append(obj.score)
-    class_names.append(obj.category)
-
-input_boxes = np.array(input_boxes)
-
-print(input_boxes)
-
-# prompt SAM image predictor to get the mask for the object
-image_predictor.set_image(np.array(image.convert("RGB")))
-
-# process the detection results
-OBJECTS = class_names
-
-print(OBJECTS)
-
-# prompt SAM 2 image predictor to get the mask for the object
-masks, scores, logits = image_predictor.predict(
-    point_coords=None,
-    point_labels=None,
-    box=input_boxes,
-    multimask_output=False,
-)
-# convert the mask shape to (n, H, W)
-if masks.ndim == 4:
-    masks = masks.squeeze(1)
-
-"""
-Step 3: Register each object's positive points to video predictor with seperate add_new_points call
-"""
-
-assert PROMPT_TYPE_FOR_VIDEO in ["point", "box", "mask"], "SAM 2 video predictor only support point/box/mask prompt"
-
-# If you are using point prompts, we uniformly sample positive points based on the mask
-if PROMPT_TYPE_FOR_VIDEO == "point":
-    # sample the positive points from mask for each objects
-    all_sample_points = sample_points_from_masks(masks=masks, num_points=10)
-
-    for object_id, (label, points) in enumerate(zip(OBJECTS, all_sample_points), start=1):
-        labels = np.ones((points.shape[0]), dtype=np.int32)
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_points_or_box(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            points=points,
-            labels=labels,
-        )
-# Using box prompt
-elif PROMPT_TYPE_FOR_VIDEO == "box":
-    for object_id, (label, box) in enumerate(zip(OBJECTS, input_boxes), start=1):
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_points_or_box(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            box=box,
-        )
-# Using mask prompt is a more straightforward way
-elif PROMPT_TYPE_FOR_VIDEO == "mask":
-    for object_id, (label, mask) in enumerate(zip(OBJECTS, masks), start=1):
-        labels = np.ones((1), dtype=np.int32)
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_mask(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            mask=mask
-        )
-else:
-    raise NotImplementedError("SAM 2 video predictor only support point/box/mask prompts")
-
-"""
-Step 4: Propagate the video predictor to get the segmentation results for each frame
-"""
-video_segments = {}  # video_segments contains the per-frame segmentation results
-for out_frame_idx, out_obj_ids, out_mask_logits in video_predictor.propagate_in_video(inference_state):
-    video_segments[out_frame_idx] = {
-        out_obj_id: (out_mask_logits[i] > 0.0).cpu().numpy()
-        for i, out_obj_id in enumerate(out_obj_ids)
-    }
-
-"""
-Step 5: Visualize the segment results across the video and save them
-"""
-
-if not os.path.exists(SAVE_TRACKING_RESULTS_DIR):
-    os.makedirs(SAVE_TRACKING_RESULTS_DIR)
-
-ID_TO_OBJECTS = {i: obj for i, obj in enumerate(OBJECTS, start=1)}
-
-for frame_idx, segments in video_segments.items():
-    img = cv2.imread(os.path.join(SOURCE_VIDEO_FRAME_DIR, frame_names[frame_idx]))
-    
-    object_ids = list(segments.keys())
-    masks = list(segments.values())
-    masks = np.concatenate(masks, axis=0)
-    
-    detections = sv.Detections(
-        xyxy=sv.mask_to_xyxy(masks),  # (n, 4)
-        mask=masks, # (n, h, w)
-        class_id=np.array(object_ids, dtype=np.int32),
-    )
-    box_annotator = sv.BoxAnnotator()
-    annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-    label_annotator = sv.LabelAnnotator()
-    annotated_frame = label_annotator.annotate(annotated_frame, detections=detections, labels=[ID_TO_OBJECTS[i] for i in object_ids])
-    mask_annotator = sv.MaskAnnotator()
-    annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-    cv2.imwrite(os.path.join(SAVE_TRACKING_RESULTS_DIR, f"annotated_frame_{frame_idx:05d}.jpg"), annotated_frame)
-
-
-"""
-Step 6: Convert the annotated frames to video
-"""
-
-create_video_from_images(SAVE_TRACKING_RESULTS_DIR, OUTPUT_VIDEO_PATH)

grounded_sam2_tracking_demo_with_continuous_id.py

@@ -1,203 +0,0 @@
-import os
-import cv2
-import torch
-import numpy as np
-import supervision as sv
-from PIL import Image
-from sam2.build_sam import build_sam2_video_predictor, build_sam2
-from sam2.sam2_image_predictor import SAM2ImagePredictor
-from transformers import AutoProcessor, AutoModelForZeroShotObjectDetection 
-from utils.track_utils import sample_points_from_masks
-from utils.video_utils import create_video_from_images
-from utils.common_utils import CommonUtils
-from utils.mask_dictionary_model import MaskDictionaryModel, ObjectInfo
-import json
-import copy
-
-"""
-Step 1: Environment settings and model initialization
-"""
-# use bfloat16 for the entire notebook
-torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__()
-
-if torch.cuda.get_device_properties(0).major >= 8:
-    # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)
-    torch.backends.cuda.matmul.allow_tf32 = True
-    torch.backends.cudnn.allow_tf32 = True
-
-# init sam image predictor and video predictor model
-sam2_checkpoint = "./checkpoints/sam2.1_hiera_large.pt"
-model_cfg = "configs/sam2.1/sam2.1_hiera_l.yaml"
-device = "cuda" if torch.cuda.is_available() else "cpu"
-print("device", device)
-
-video_predictor = build_sam2_video_predictor(model_cfg, sam2_checkpoint)
-sam2_image_model = build_sam2(model_cfg, sam2_checkpoint, device=device)
-image_predictor = SAM2ImagePredictor(sam2_image_model)
-
-
-# init grounding dino model from huggingface
-model_id = "IDEA-Research/grounding-dino-tiny"
-processor = AutoProcessor.from_pretrained(model_id)
-grounding_model = AutoModelForZeroShotObjectDetection.from_pretrained(model_id).to(device)
-
-
-# setup the input image and text prompt for SAM 2 and Grounding DINO
-# VERY important: text queries need to be lowercased + end with a dot
-text = "car."
-
-# `video_dir` a directory of JPEG frames with filenames like `<frame_index>.jpg`  
-video_dir = "notebooks/videos/car"
-# 'output_dir' is the directory to save the annotated frames
-output_dir = "./outputs"
-# 'output_video_path' is the path to save the final video
-output_video_path = "./outputs/output.mp4"
-# create the output directory
-CommonUtils.creat_dirs(output_dir)
-mask_data_dir = os.path.join(output_dir, "mask_data")
-json_data_dir = os.path.join(output_dir, "json_data")
-result_dir = os.path.join(output_dir, "result")
-CommonUtils.creat_dirs(mask_data_dir)
-CommonUtils.creat_dirs(json_data_dir)
-# scan all the JPEG frame names in this directory
-frame_names = [
-    p for p in os.listdir(video_dir)
-    if os.path.splitext(p)[-1] in [".jpg", ".jpeg", ".JPG", ".JPEG", ".png", ".PNG"]
-]
-frame_names.sort(key=lambda p: int(os.path.splitext(p)[0]))
-
-# init video predictor state
-inference_state = video_predictor.init_state(video_path=video_dir, offload_video_to_cpu=True, async_loading_frames=True)
-step = 20 # the step to sample frames for Grounding DINO predictor
-
-sam2_masks = MaskDictionaryModel()
-PROMPT_TYPE_FOR_VIDEO = "mask" # box, mask or point
-objects_count = 0
-
-"""
-Step 2: Prompt Grounding DINO and SAM image predictor to get the box and mask for all frames
-"""
-print("Total frames:", len(frame_names))
-for start_frame_idx in range(0, len(frame_names), step):
-# prompt grounding dino to get the box coordinates on specific frame
-    print("start_frame_idx", start_frame_idx)
-    # continue
-    img_path = os.path.join(video_dir, frame_names[start_frame_idx])
-    image = Image.open(img_path)
-    image_base_name = frame_names[start_frame_idx].split(".")[0]
-    mask_dict = MaskDictionaryModel(promote_type = PROMPT_TYPE_FOR_VIDEO, mask_name = f"mask_{image_base_name}.npy")
-
-    # run Grounding DINO on the image
-    inputs = processor(images=image, text=text, return_tensors="pt").to(device)
-    with torch.no_grad():
-        outputs = grounding_model(**inputs)
-
-    results = processor.post_process_grounded_object_detection(
-        outputs,
-        inputs.input_ids,
-        box_threshold=0.25,
-        text_threshold=0.25,
-        target_sizes=[image.size[::-1]]
-    )
-
-    # prompt SAM image predictor to get the mask for the object
-    image_predictor.set_image(np.array(image.convert("RGB")))
-
-    # process the detection results
-    input_boxes = results[0]["boxes"] # .cpu().numpy()
-    # print("results[0]",results[0])
-    OBJECTS = results[0]["labels"]
-    if input_boxes.shape[0] != 0:
-        # prompt SAM 2 image predictor to get the mask for the object
-        masks, scores, logits = image_predictor.predict(
-            point_coords=None,
-            point_labels=None,
-            box=input_boxes,
-            multimask_output=False,
-        )
-        # convert the mask shape to (n, H, W)
-        if masks.ndim == 2:
-            masks = masks[None]
-            scores = scores[None]
-            logits = logits[None]
-        elif masks.ndim == 4:
-            masks = masks.squeeze(1)
-
-        """
-        Step 3: Register each object's positive points to video predictor
-        """
-
-        # If you are using point prompts, we uniformly sample positive points based on the mask
-        if mask_dict.promote_type == "mask":
-            mask_dict.add_new_frame_annotation(mask_list=torch.tensor(masks).to(device), box_list=torch.tensor(input_boxes), label_list=OBJECTS)
-        else:
-            raise NotImplementedError("SAM 2 video predictor only support mask prompts")
-
-
-        """
-        Step 4: Propagate the video predictor to get the segmentation results for each frame
-        """
-        objects_count = mask_dict.update_masks(tracking_annotation_dict=sam2_masks, iou_threshold=0.8, objects_count=objects_count)
-        print("objects_count", objects_count)
-    else:
-        print("No object detected in the frame, skip merge the frame merge {}".format(frame_names[start_frame_idx]))
-        mask_dict = sam2_masks
-
-    
-    if len(mask_dict.labels) == 0:
-        mask_dict.save_empty_mask_and_json(mask_data_dir, json_data_dir, image_name_list = frame_names[start_frame_idx:start_frame_idx+step])
-        print("No object detected in the frame, skip the frame {}".format(start_frame_idx))
-        continue
-    else: 
-        video_predictor.reset_state(inference_state)
-
-        for object_id, object_info in mask_dict.labels.items():
-            frame_idx, out_obj_ids, out_mask_logits = video_predictor.add_new_mask(
-                    inference_state,
-                    start_frame_idx,
-                    object_id,
-                    object_info.mask,
-                )
-        
-        video_segments = {}  # output the following {step} frames tracking masks
-        for out_frame_idx, out_obj_ids, out_mask_logits in video_predictor.propagate_in_video(inference_state, max_frame_num_to_track=step, start_frame_idx=start_frame_idx):
-            frame_masks = MaskDictionaryModel()
-            
-            for i, out_obj_id in enumerate(out_obj_ids):
-                out_mask = (out_mask_logits[i] > 0.0) # .cpu().numpy()
-                object_info = ObjectInfo(instance_id = out_obj_id, mask = out_mask[0], class_name = mask_dict.get_target_class_name(out_obj_id))
-                object_info.update_box()
-                frame_masks.labels[out_obj_id] = object_info
-                image_base_name = frame_names[out_frame_idx].split(".")[0]
-                frame_masks.mask_name = f"mask_{image_base_name}.npy"
-                frame_masks.mask_height = out_mask.shape[-2]
-                frame_masks.mask_width = out_mask.shape[-1]
-
-            video_segments[out_frame_idx] = frame_masks
-            sam2_masks = copy.deepcopy(frame_masks)
-
-        print("video_segments:", len(video_segments))
-    """
-    Step 5: save the tracking masks and json files
-    """
-    for frame_idx, frame_masks_info in video_segments.items():
-        mask = frame_masks_info.labels
-        mask_img = torch.zeros(frame_masks_info.mask_height, frame_masks_info.mask_width)
-        for obj_id, obj_info in mask.items():
-            mask_img[obj_info.mask == True] = obj_id
-
-        mask_img = mask_img.numpy().astype(np.uint16)
-        np.save(os.path.join(mask_data_dir, frame_masks_info.mask_name), mask_img)
-
-        json_data = frame_masks_info.to_dict()
-        json_data_path = os.path.join(json_data_dir, frame_masks_info.mask_name.replace(".npy", ".json"))
-        with open(json_data_path, "w") as f:
-            json.dump(json_data, f)
-
-
-"""
-Step 6: Draw the results and save the video
-"""
-CommonUtils.draw_masks_and_box_with_supervision(video_dir, mask_data_dir, json_data_dir, result_dir)
-
-create_video_from_images(result_dir, output_video_path, frame_rate=15)

grounded_sam2_tracking_demo_with_continuous_id_gd1.5.py

@@ -1,224 +0,0 @@
-# dds cloudapi for Grounding DINO 1.5
-from dds_cloudapi_sdk import Config
-from dds_cloudapi_sdk import Client
-from dds_cloudapi_sdk import DetectionTask
-from dds_cloudapi_sdk import TextPrompt
-from dds_cloudapi_sdk import DetectionModel
-from dds_cloudapi_sdk import DetectionTarget
-
-
-import os
-import torch
-import numpy as np
-from PIL import Image
-from sam2.build_sam import build_sam2_video_predictor, build_sam2
-from sam2.sam2_image_predictor import SAM2ImagePredictor
-from transformers import AutoProcessor, AutoModelForZeroShotObjectDetection 
-from utils.video_utils import create_video_from_images
-from utils.common_utils import CommonUtils
-from utils.mask_dictionary_model import MaskDictionaryModel, ObjectInfo
-import json
-import copy
-
-"""
-Step 1: Environment settings and model initialization
-"""
-# use bfloat16 for the entire notebook
-torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__()
-
-if torch.cuda.get_device_properties(0).major >= 8:
-    # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)
-    torch.backends.cuda.matmul.allow_tf32 = True
-    torch.backends.cudnn.allow_tf32 = True
-
-# init sam image predictor and video predictor model
-sam2_checkpoint = "./checkpoints/sam2.1_hiera_large.pt"
-model_cfg = "configs/sam2.1/sam2.1_hiera_l.yaml"
-device = "cuda" if torch.cuda.is_available() else "cpu"
-print("device", device)
-
-video_predictor = build_sam2_video_predictor(model_cfg, sam2_checkpoint)
-sam2_image_model = build_sam2(model_cfg, sam2_checkpoint, device=device)
-image_predictor = SAM2ImagePredictor(sam2_image_model)
-
-
-# init grounding dino model from huggingface
-model_id = "IDEA-Research/grounding-dino-tiny"
-processor = AutoProcessor.from_pretrained(model_id)
-grounding_model = AutoModelForZeroShotObjectDetection.from_pretrained(model_id).to(device)
-
-
-# setup the input image and text prompt for SAM 2 and Grounding DINO
-# VERY important: text queries need to be lowercased + end with a dot
-text = "car."
-
-# `video_dir` a directory of JPEG frames with filenames like `<frame_index>.jpg`  
-video_dir = "notebooks/videos/car"
-# 'output_dir' is the directory to save the annotated frames
-output_dir = "./outputs"
-# 'output_video_path' is the path to save the final video
-output_video_path = "./outputs/output.mp4"
-# create the output directory
-CommonUtils.creat_dirs(output_dir)
-mask_data_dir = os.path.join(output_dir, "mask_data")
-json_data_dir = os.path.join(output_dir, "json_data")
-result_dir = os.path.join(output_dir, "result")
-CommonUtils.creat_dirs(mask_data_dir)
-CommonUtils.creat_dirs(json_data_dir)
-# scan all the JPEG frame names in this directory
-frame_names = [
-    p for p in os.listdir(video_dir)
-    if os.path.splitext(p)[-1] in [".jpg", ".jpeg", ".JPG", ".JPEG", ".png", ".PNG"]
-]
-frame_names.sort(key=lambda p: int(os.path.splitext(p)[0]))
-
-# init video predictor state
-inference_state = video_predictor.init_state(video_path=video_dir)
-step = 10 # the step to sample frames for Grounding DINO predictor
-
-sam2_masks = MaskDictionaryModel()
-PROMPT_TYPE_FOR_VIDEO = "mask" # box, mask or point
-objects_count = 0
-
-"""
-Step 2: Prompt Grounding DINO and SAM image predictor to get the box and mask for all frames
-"""
-print("Total frames:", len(frame_names))
-for start_frame_idx in range(0, len(frame_names), step):
-# prompt grounding dino to get the box coordinates on specific frame
-    print("start_frame_idx", start_frame_idx)
-    # continue
-    img_path = os.path.join(video_dir, frame_names[start_frame_idx])
-    image = Image.open(img_path)
-    image_base_name = frame_names[start_frame_idx].split(".")[0]
-    mask_dict = MaskDictionaryModel(promote_type = PROMPT_TYPE_FOR_VIDEO, mask_name = f"mask_{image_base_name}.npy")
-
-    # run Grounding DINO 1.5 on the image
-
-    API_TOKEN_FOR_GD1_5 = "Your API token"
-
-    config = Config(API_TOKEN_FOR_GD1_5)
-    # Step 2: initialize the client
-    client = Client(config)
-    
-    image_url = client.upload_file(img_path)
-    task = DetectionTask(
-        image_url=image_url,
-        prompts=[TextPrompt(text=text)],
-        targets=[DetectionTarget.BBox],  # detect bbox
-        model=DetectionModel.GDino1_6_Pro,  # detect with GroundingDino-1.5-Pro model
-    )
-    client.run_task(task)
-    result = task.result
-
-    objects = result.objects  # the list of detected objects
-    input_boxes = []
-    confidences = []
-    class_names = []
-
-    for idx, obj in enumerate(objects):
-        input_boxes.append(obj.bbox)
-        confidences.append(obj.score)
-        class_names.append(obj.category)
-
-    input_boxes = np.array(input_boxes)
-    OBJECTS = class_names
-    if input_boxes.shape[0] != 0:
-        # prompt SAM image predictor to get the mask for the object
-        image_predictor.set_image(np.array(image.convert("RGB")))
-
-        # prompt SAM 2 image predictor to get the mask for the object
-        masks, scores, logits = image_predictor.predict(
-            point_coords=None,
-            point_labels=None,
-            box=input_boxes,
-            multimask_output=False,
-        )
-        # convert the mask shape to (n, H, W)
-        if masks.ndim == 2:
-            masks = masks[None]
-            scores = scores[None]
-            logits = logits[None]
-        elif masks.ndim == 4:
-            masks = masks.squeeze(1)
-
-        """
-        Step 3: Register each object's positive points to video predictor
-        """
-
-        # If you are using point prompts, we uniformly sample positive points based on the mask
-        if mask_dict.promote_type == "mask":
-            mask_dict.add_new_frame_annotation(mask_list=torch.tensor(masks).to(device), box_list=torch.tensor(input_boxes), label_list=OBJECTS)
-        else:
-            raise NotImplementedError("SAM 2 video predictor only support mask prompts")
-
-
-        
-        objects_count = mask_dict.update_masks(tracking_annotation_dict=sam2_masks, iou_threshold=0.8, objects_count=objects_count)
-        print("objects_count", objects_count)
-    
-    else:
-        print("No object detected in the frame, skip merge the frame merge {}".format(frame_names[start_frame_idx]))
-        mask_dict = sam2_masks
-
-    
-    """
-        Step 4: Propagate the video predictor to get the segmentation results for each frame
-    """
-    if len(mask_dict.labels) == 0:
-        mask_dict.save_empty_mask_and_json(mask_data_dir, json_data_dir, image_name_list = frame_names[start_frame_idx:start_frame_idx+step])
-        print("No object detected in the frame, skip the frame {}".format(start_frame_idx))
-        continue
-    else:
-        video_predictor.reset_state(inference_state)
-
-        for object_id, object_info in mask_dict.labels.items():
-            frame_idx, out_obj_ids, out_mask_logits = video_predictor.add_new_mask(
-                    inference_state,
-                    start_frame_idx,
-                    object_id,
-                    object_info.mask,
-                )
-        
-        video_segments = {}  # output the following {step} frames tracking masks
-        for out_frame_idx, out_obj_ids, out_mask_logits in video_predictor.propagate_in_video(inference_state, max_frame_num_to_track=step, start_frame_idx=start_frame_idx):
-            frame_masks = MaskDictionaryModel()
-            
-            for i, out_obj_id in enumerate(out_obj_ids):
-                out_mask = (out_mask_logits[i] > 0.0) # .cpu().numpy()
-                object_info = ObjectInfo(instance_id = out_obj_id, mask = out_mask[0], class_name = mask_dict.get_target_class_name(out_obj_id))
-                object_info.update_box()
-                frame_masks.labels[out_obj_id] = object_info
-                image_base_name = frame_names[out_frame_idx].split(".")[0]
-                frame_masks.mask_name = f"mask_{image_base_name}.npy"
-                frame_masks.mask_height = out_mask.shape[-2]
-                frame_masks.mask_width = out_mask.shape[-1]
-
-            video_segments[out_frame_idx] = frame_masks
-            sam2_masks = copy.deepcopy(frame_masks)
-
-        print("video_segments:", len(video_segments))
-    """
-    Step 5: save the tracking masks and json files
-    """
-    for frame_idx, frame_masks_info in video_segments.items():
-        mask = frame_masks_info.labels
-        mask_img = torch.zeros(frame_masks_info.mask_height, frame_masks_info.mask_width)
-        for obj_id, obj_info in mask.items():
-            mask_img[obj_info.mask == True] = obj_id
-
-        mask_img = mask_img.numpy().astype(np.uint16)
-        np.save(os.path.join(mask_data_dir, frame_masks_info.mask_name), mask_img)
-
-        json_data = frame_masks_info.to_dict()
-        json_data_path = os.path.join(json_data_dir, frame_masks_info.mask_name.replace(".npy", ".json"))
-        with open(json_data_path, "w") as f:
-            json.dump(json_data, f)
-
-
-"""
-Step 6: Draw the results and save the video
-"""
-CommonUtils.draw_masks_and_box_with_supervision(video_dir, mask_data_dir, json_data_dir, result_dir)
-
-create_video_from_images(result_dir, output_video_path, frame_rate=30)

grounded_sam2_tracking_demo_with_continuous_id_plus.py

@@ -1,247 +0,0 @@
-import os
-import cv2
-import torch
-import numpy as np
-import supervision as sv
-from PIL import Image
-from sam2.build_sam import build_sam2_video_predictor, build_sam2
-from sam2.sam2_image_predictor import SAM2ImagePredictor
-from transformers import AutoProcessor, AutoModelForZeroShotObjectDetection 
-from utils.track_utils import sample_points_from_masks
-from utils.video_utils import create_video_from_images
-from utils.common_utils import CommonUtils
-from utils.mask_dictionary_model import MaskDictionaryModel, ObjectInfo
-import json
-import copy
-
-# This demo shows the continuous object tracking plus reverse tracking with Grounding DINO and SAM 2
-"""
-Step 1: Environment settings and model initialization
-"""
-# use bfloat16 for the entire notebook
-torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__()
-
-if torch.cuda.get_device_properties(0).major >= 8:
-    # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)
-    torch.backends.cuda.matmul.allow_tf32 = True
-    torch.backends.cudnn.allow_tf32 = True
-
-# init sam image predictor and video predictor model
-sam2_checkpoint = "./checkpoints/sam2.1_hiera_large.pt"
-model_cfg = "configs/sam2.1/sam2.1_hiera_l.yaml"
-device = "cuda" if torch.cuda.is_available() else "cpu"
-print("device", device)
-
-video_predictor = build_sam2_video_predictor(model_cfg, sam2_checkpoint)
-sam2_image_model = build_sam2(model_cfg, sam2_checkpoint, device=device)
-image_predictor = SAM2ImagePredictor(sam2_image_model)
-
-
-# init grounding dino model from huggingface
-model_id = "IDEA-Research/grounding-dino-tiny"
-processor = AutoProcessor.from_pretrained(model_id)
-grounding_model = AutoModelForZeroShotObjectDetection.from_pretrained(model_id).to(device)
-
-
-# setup the input image and text prompt for SAM 2 and Grounding DINO
-# VERY important: text queries need to be lowercased + end with a dot
-text = "car."
-
-# `video_dir` a directory of JPEG frames with filenames like `<frame_index>.jpg`  
-video_dir = "notebooks/videos/car"
-# 'output_dir' is the directory to save the annotated frames
-output_dir = "outputs"
-# 'output_video_path' is the path to save the final video
-output_video_path = "./outputs/output.mp4"
-# create the output directory
-mask_data_dir = os.path.join(output_dir, "mask_data")
-json_data_dir = os.path.join(output_dir, "json_data")
-result_dir = os.path.join(output_dir, "result")
-CommonUtils.creat_dirs(mask_data_dir)
-CommonUtils.creat_dirs(json_data_dir)
-# scan all the JPEG frame names in this directory
-frame_names = [
-    p for p in os.listdir(video_dir)
-    if os.path.splitext(p)[-1] in [".jpg", ".jpeg", ".JPG", ".JPEG", ".png", ".PNG"]
-]
-frame_names.sort(key=lambda p: int(os.path.splitext(p)[0]))
-
-# init video predictor state
-inference_state = video_predictor.init_state(video_path=video_dir)
-step = 20 # the step to sample frames for Grounding DINO predictor
-
-sam2_masks = MaskDictionaryModel()
-PROMPT_TYPE_FOR_VIDEO = "mask" # box, mask or point
-objects_count = 0
-frame_object_count = {}
-"""
-Step 2: Prompt Grounding DINO and SAM image predictor to get the box and mask for all frames
-"""
-print("Total frames:", len(frame_names))
-for start_frame_idx in range(0, len(frame_names), step):
-# prompt grounding dino to get the box coordinates on specific frame
-    print("start_frame_idx", start_frame_idx)
-    # continue
-    img_path = os.path.join(video_dir, frame_names[start_frame_idx])
-    image = Image.open(img_path).convert("RGB")
-    image_base_name = frame_names[start_frame_idx].split(".")[0]
-    mask_dict = MaskDictionaryModel(promote_type = PROMPT_TYPE_FOR_VIDEO, mask_name = f"mask_{image_base_name}.npy")
-
-    # run Grounding DINO on the image
-    inputs = processor(images=image, text=text, return_tensors="pt").to(device)
-    with torch.no_grad():
-        outputs = grounding_model(**inputs)
-
-    results = processor.post_process_grounded_object_detection(
-        outputs,
-        inputs.input_ids,
-        box_threshold=0.25,
-        text_threshold=0.25,
-        target_sizes=[image.size[::-1]]
-    )
-
-    # prompt SAM image predictor to get the mask for the object
-    image_predictor.set_image(np.array(image.convert("RGB")))
-
-    # process the detection results
-    input_boxes = results[0]["boxes"] # .cpu().numpy()
-    # print("results[0]",results[0])
-    OBJECTS = results[0]["labels"]
-    if input_boxes.shape[0] != 0:
-
-        # prompt SAM 2 image predictor to get the mask for the object
-        masks, scores, logits = image_predictor.predict(
-            point_coords=None,
-            point_labels=None,
-            box=input_boxes,
-            multimask_output=False,
-        )
-        # convert the mask shape to (n, H, W)
-        if masks.ndim == 2:
-            masks = masks[None]
-            scores = scores[None]
-            logits = logits[None]
-        elif masks.ndim == 4:
-            masks = masks.squeeze(1)
-        """
-        Step 3: Register each object's positive points to video predictor
-        """
-
-        # If you are using point prompts, we uniformly sample positive points based on the mask
-        if mask_dict.promote_type == "mask":
-            mask_dict.add_new_frame_annotation(mask_list=torch.tensor(masks).to(device), box_list=torch.tensor(input_boxes), label_list=OBJECTS)
-        else:
-            raise NotImplementedError("SAM 2 video predictor only support mask prompts")
-    else:
-        print("No object detected in the frame, skip merge the frame merge {}".format(frame_names[start_frame_idx]))
-        mask_dict = sam2_masks
-
-    """
-    Step 4: Propagate the video predictor to get the segmentation results for each frame
-    """
-    objects_count = mask_dict.update_masks(tracking_annotation_dict=sam2_masks, iou_threshold=0.8, objects_count=objects_count)
-    frame_object_count[start_frame_idx] = objects_count
-    print("objects_count", objects_count)
-    
-    if len(mask_dict.labels) == 0:
-        mask_dict.save_empty_mask_and_json(mask_data_dir, json_data_dir, image_name_list = frame_names[start_frame_idx:start_frame_idx+step])
-        print("No object detected in the frame, skip the frame {}".format(start_frame_idx))
-        continue
-    else:
-        video_predictor.reset_state(inference_state)
-
-        for object_id, object_info in mask_dict.labels.items():
-            frame_idx, out_obj_ids, out_mask_logits = video_predictor.add_new_mask(
-                    inference_state,
-                    start_frame_idx,
-                    object_id,
-                    object_info.mask,
-                )
-        
-        video_segments = {}  # output the following {step} frames tracking masks
-        for out_frame_idx, out_obj_ids, out_mask_logits in video_predictor.propagate_in_video(inference_state, max_frame_num_to_track=step, start_frame_idx=start_frame_idx):
-            frame_masks = MaskDictionaryModel()
-            
-            for i, out_obj_id in enumerate(out_obj_ids):
-                out_mask = (out_mask_logits[i] > 0.0) # .cpu().numpy()
-                object_info = ObjectInfo(instance_id = out_obj_id, mask = out_mask[0], class_name = mask_dict.get_target_class_name(out_obj_id), logit=mask_dict.get_target_logit(out_obj_id))
-                object_info.update_box()
-                frame_masks.labels[out_obj_id] = object_info
-                image_base_name = frame_names[out_frame_idx].split(".")[0]
-                frame_masks.mask_name = f"mask_{image_base_name}.npy"
-                frame_masks.mask_height = out_mask.shape[-2]
-                frame_masks.mask_width = out_mask.shape[-1]
-
-            video_segments[out_frame_idx] = frame_masks
-            sam2_masks = copy.deepcopy(frame_masks)
-
-        print("video_segments:", len(video_segments))
-    """
-    Step 5: save the tracking masks and json files
-    """
-    for frame_idx, frame_masks_info in video_segments.items():
-        mask = frame_masks_info.labels
-        mask_img = torch.zeros(frame_masks_info.mask_height, frame_masks_info.mask_width)
-        for obj_id, obj_info in mask.items():
-            mask_img[obj_info.mask == True] = obj_id
-
-        mask_img = mask_img.numpy().astype(np.uint16)
-        np.save(os.path.join(mask_data_dir, frame_masks_info.mask_name), mask_img)
-
-        json_data_path = os.path.join(json_data_dir, frame_masks_info.mask_name.replace(".npy", ".json"))
-        frame_masks_info.to_json(json_data_path)
-       
-
-CommonUtils.draw_masks_and_box_with_supervision(video_dir, mask_data_dir, json_data_dir, result_dir)
-
-print("try reverse tracking")
-start_object_id = 0
-object_info_dict = {}
-for frame_idx, current_object_count in frame_object_count.items():
-    print("reverse tracking frame", frame_idx, frame_names[frame_idx])
-    if frame_idx != 0:
-        video_predictor.reset_state(inference_state)
-        image_base_name = frame_names[frame_idx].split(".")[0]
-        json_data_path = os.path.join(json_data_dir, f"mask_{image_base_name}.json")
-        json_data = MaskDictionaryModel().from_json(json_data_path)
-        mask_data_path = os.path.join(mask_data_dir, f"mask_{image_base_name}.npy")
-        mask_array = np.load(mask_data_path)
-        for object_id in range(start_object_id+1, current_object_count+1):
-            print("reverse tracking object", object_id)
-            object_info_dict[object_id] = json_data.labels[object_id]
-            video_predictor.add_new_mask(inference_state, frame_idx, object_id, mask_array == object_id)
-    start_object_id = current_object_count
-        
-    
-    for out_frame_idx, out_obj_ids, out_mask_logits in video_predictor.propagate_in_video(inference_state, max_frame_num_to_track=step*2,  start_frame_idx=frame_idx, reverse=True):
-        image_base_name = frame_names[out_frame_idx].split(".")[0]
-        json_data_path = os.path.join(json_data_dir, f"mask_{image_base_name}.json")
-        json_data = MaskDictionaryModel().from_json(json_data_path)
-        mask_data_path = os.path.join(mask_data_dir, f"mask_{image_base_name}.npy")
-        mask_array = np.load(mask_data_path)
-        # merge the reverse tracking masks with the original masks
-        for i, out_obj_id in enumerate(out_obj_ids):
-            out_mask = (out_mask_logits[i] > 0.0).cpu()
-            if out_mask.sum() == 0:
-                print("no mask for object", out_obj_id, "at frame", out_frame_idx)
-                continue
-            object_info = object_info_dict[out_obj_id]
-            object_info.mask = out_mask[0]
-            object_info.update_box()
-            json_data.labels[out_obj_id] = object_info
-            mask_array = np.where(mask_array != out_obj_id, mask_array, 0)
-            mask_array[object_info.mask] = out_obj_id
-        
-        np.save(mask_data_path, mask_array)
-        json_data.to_json(json_data_path)
-
-        
-
-
-
-"""
-Step 6: Draw the results and save the video
-"""
-CommonUtils.draw_masks_and_box_with_supervision(video_dir, mask_data_dir, json_data_dir, result_dir+"_reverse")
-
-create_video_from_images(result_dir, output_video_path, frame_rate=15)

grounded_sam2_tracking_demo_with_gd1.5.py

@@ -1,221 +0,0 @@
-# dds cloudapi for Grounding DINO 1.5
-from dds_cloudapi_sdk import Config
-from dds_cloudapi_sdk import Client
-from dds_cloudapi_sdk import DetectionTask
-from dds_cloudapi_sdk import TextPrompt
-from dds_cloudapi_sdk import DetectionModel
-from dds_cloudapi_sdk import DetectionTarget
-
-import os
-import cv2
-import torch
-import numpy as np
-import supervision as sv
-from PIL import Image
-from sam2.build_sam import build_sam2_video_predictor, build_sam2
-from sam2.sam2_image_predictor import SAM2ImagePredictor 
-from utils.track_utils import sample_points_from_masks
-from utils.video_utils import create_video_from_images
-
-
-"""
-Step 1: Environment settings and model initialization for SAM 2
-"""
-# use bfloat16 for the entire notebook
-torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__()
-
-if torch.cuda.get_device_properties(0).major >= 8:
-    # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)
-    torch.backends.cuda.matmul.allow_tf32 = True
-    torch.backends.cudnn.allow_tf32 = True
-
-# init sam image predictor and video predictor model
-sam2_checkpoint = "./checkpoints/sam2.1_hiera_large.pt"
-model_cfg = "configs/sam2.1/sam2.1_hiera_l.yaml"
-
-video_predictor = build_sam2_video_predictor(model_cfg, sam2_checkpoint)
-sam2_image_model = build_sam2(model_cfg, sam2_checkpoint)
-image_predictor = SAM2ImagePredictor(sam2_image_model)
-
-
-# `video_dir` a directory of JPEG frames with filenames like `<frame_index>.jpg`
-video_dir = "notebooks/videos/bedroom"
-
-# scan all the JPEG frame names in this directory
-frame_names = [
-    p for p in os.listdir(video_dir)
-    if os.path.splitext(p)[-1] in [".jpg", ".jpeg", ".JPG", ".JPEG", ".png", ".PNG"]
-]
-frame_names.sort(key=lambda p: int(os.path.splitext(p)[0]))
-
-# init video predictor state
-inference_state = video_predictor.init_state(video_path=video_dir)
-
-ann_frame_idx = 0  # the frame index we interact with
-ann_obj_id = 1  # give a unique id to each object we interact with (it can be any integers)
-
-
-"""
-Step 2: Prompt Grounding DINO 1.5 with Cloud API for box coordinates
-"""
-
-# prompt grounding dino to get the box coordinates on specific frame
-img_path = os.path.join(video_dir, frame_names[ann_frame_idx])
-image = Image.open(img_path)
-
-# Step 1: initialize the config
-token = "Your API token"
-config = Config(token)
-
-# Step 2: initialize the client
-client = Client(config)
-
-# Step 3: run the task by DetectionTask class
-# image_url = "https://algosplt.oss-cn-shenzhen.aliyuncs.com/test_files/tasks/detection/iron_man.jpg"
-# if you are processing local image file, upload them to DDS server to get the image url
-image_url = client.upload_file(img_path)
-
-task = DetectionTask(
-    image_url=image_url,
-    prompts=[TextPrompt(text="children. pillow")],
-    targets=[DetectionTarget.BBox],  # detect bbox
-    model=DetectionModel.GDino1_5_Pro,  # detect with GroundingDino-1.5-Pro model
-    bbox_threshold=0.2,
-)
-
-client.run_task(task)
-result = task.result
-
-objects = result.objects  # the list of detected objects
-
-
-input_boxes = []
-confidences = []
-class_names = []
-
-for idx, obj in enumerate(objects):
-    input_boxes.append(obj.bbox)
-    confidences.append(obj.score)
-    class_names.append(obj.category)
-
-input_boxes = np.array(input_boxes)
-
-print(input_boxes)
-
-# prompt SAM image predictor to get the mask for the object
-image_predictor.set_image(np.array(image.convert("RGB")))
-
-# process the detection results
-OBJECTS = class_names
-
-print(OBJECTS)
-
-# prompt SAM 2 image predictor to get the mask for the object
-masks, scores, logits = image_predictor.predict(
-    point_coords=None,
-    point_labels=None,
-    box=input_boxes,
-    multimask_output=False,
-)
-
-# convert the mask shape to (n, H, W)
-if masks.ndim == 3:
-    masks = masks[None]
-    scores = scores[None]
-    logits = logits[None]
-elif masks.ndim == 4:
-    masks = masks.squeeze(1)
-
-"""
-Step 3: Register each object's positive points to video predictor with seperate add_new_points call
-"""
-
-PROMPT_TYPE_FOR_VIDEO = "box" # or "point"
-
-assert PROMPT_TYPE_FOR_VIDEO in ["point", "box", "mask"], "SAM 2 video predictor only support point/box/mask prompt"
-
-# If you are using point prompts, we uniformly sample positive points based on the mask
-if PROMPT_TYPE_FOR_VIDEO == "point":
-    # sample the positive points from mask for each objects
-    all_sample_points = sample_points_from_masks(masks=masks, num_points=10)
-
-    for object_id, (label, points) in enumerate(zip(OBJECTS, all_sample_points), start=1):
-        labels = np.ones((points.shape[0]), dtype=np.int32)
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_points_or_box(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            points=points,
-            labels=labels,
-        )
-# Using box prompt
-elif PROMPT_TYPE_FOR_VIDEO == "box":
-    for object_id, (label, box) in enumerate(zip(OBJECTS, input_boxes), start=1):
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_points_or_box(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            box=box,
-        )
-# Using mask prompt is a more straightforward way
-elif PROMPT_TYPE_FOR_VIDEO == "mask":
-    for object_id, (label, mask) in enumerate(zip(OBJECTS, masks), start=1):
-        labels = np.ones((1), dtype=np.int32)
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_mask(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            mask=mask
-        )
-else:
-    raise NotImplementedError("SAM 2 video predictor only support point/box/mask prompts")
-
-
-
-"""
-Step 4: Propagate the video predictor to get the segmentation results for each frame
-"""
-video_segments = {}  # video_segments contains the per-frame segmentation results
-for out_frame_idx, out_obj_ids, out_mask_logits in video_predictor.propagate_in_video(inference_state):
-    video_segments[out_frame_idx] = {
-        out_obj_id: (out_mask_logits[i] > 0.0).cpu().numpy()
-        for i, out_obj_id in enumerate(out_obj_ids)
-    }
-
-"""
-Step 5: Visualize the segment results across the video and save them
-"""
-
-save_dir = "./tracking_results"
-
-if not os.path.exists(save_dir):
-    os.makedirs(save_dir)
-
-ID_TO_OBJECTS = {i: obj for i, obj in enumerate(OBJECTS, start=1)}
-for frame_idx, segments in video_segments.items():
-    img = cv2.imread(os.path.join(video_dir, frame_names[frame_idx]))
-    
-    object_ids = list(segments.keys())
-    masks = list(segments.values())
-    masks = np.concatenate(masks, axis=0)
-    
-    detections = sv.Detections(
-        xyxy=sv.mask_to_xyxy(masks),  # (n, 4)
-        mask=masks, # (n, h, w)
-        class_id=np.array(object_ids, dtype=np.int32),
-    )
-    box_annotator = sv.BoxAnnotator()
-    annotated_frame = box_annotator.annotate(scene=img.copy(), detections=detections)
-    label_annotator = sv.LabelAnnotator()
-    annotated_frame = label_annotator.annotate(annotated_frame, detections=detections, labels=[ID_TO_OBJECTS[i] for i in object_ids])
-    mask_annotator = sv.MaskAnnotator()
-    annotated_frame = mask_annotator.annotate(scene=annotated_frame, detections=detections)
-    cv2.imwrite(os.path.join(save_dir, f"annotated_frame_{frame_idx:05d}.jpg"), annotated_frame)
-
-
-"""
-Step 6: Convert the annotated frames to video
-"""
-
-output_video_path = "./children_tracking_demo_video.mp4"
-create_video_from_images(save_dir, output_video_path)

grounded_samurai_dinox.py

@@ -1,30 +1,29 @@
-# dds cloudapi for Grounding DINO 1.5
+# libraries for SAMURAI
+import os
+import cv2
+import torch
+import numpy as np
+import supervision as sv
+import sys
+from pathlib import Path
+from tqdm import tqdm
+from PIL import Image
+sys.path.append("./sam2")
+from sam2.build_sam import build_sam2_video_predictor
+
+# dds cloudapi for DINO-X
 from dds_cloudapi_sdk import Config
 from dds_cloudapi_sdk import Client
 from dds_cloudapi_sdk.tasks.dinox import DinoxTask
 from dds_cloudapi_sdk.tasks.types import DetectionTarget
 from dds_cloudapi_sdk import TextPrompt
 
-import os
-import cv2
-import torch
-import numpy as np
-import supervision as sv
-
-from pathlib import Path
-from tqdm import tqdm
-from PIL import Image
-from sam2.build_sam import build_sam2_video_predictor, build_sam2
-from sam2.sam2_image_predictor import SAM2ImagePredictor 
-from utils.track_utils import sample_points_from_masks
-from utils.video_utils import create_video_from_images
-
 """
 Hyperparam for Ground and Tracking
 """
-VIDEO_PATH = "./assets/hippopotamus.mp4"
-TEXT_PROMPT = "hippopotamus."
-OUTPUT_VIDEO_PATH = "./hippopotamus_tracking_demo.mp4"
+VIDEO_PATH = "demo.mp4"
+TEXT_PROMPT = "person."
+OUTPUT_VIDEO_PATH = "./tracking_demo.mp4"
 SOURCE_VIDEO_FRAME_DIR = "./custom_video_frames"
 SAVE_TRACKING_RESULTS_DIR = "./tracking_results"
 API_TOKEN_FOR_DINOX = "Your API token"
@@ -43,13 +42,10 @@ if torch.cuda.get_device_properties(0).major >= 8:
     torch.backends.cudnn.allow_tf32 = True
 
 # init sam image predictor and video predictor model
-sam2_checkpoint = "./checkpoints/sam2.1_hiera_large.pt"
-model_cfg = "configs/sam2.1/sam2.1_hiera_l.yaml"
+sam2_checkpoint = "/comp_robot/rentianhe/code/samurai/sam2/checkpoints/sam2.1_hiera_large.pt"
+model_cfg = "configs/samurai/sam2.1_hiera_l.yaml"
 
 video_predictor = build_sam2_video_predictor(model_cfg, sam2_checkpoint)
-sam2_image_model = build_sam2(model_cfg, sam2_checkpoint)
-image_predictor = SAM2ImagePredictor(sam2_image_model)
-
 
 # # `video_dir` a directory of JPEG frames with filenames like `<frame_index>.jpg`
 # video_dir = "notebooks/videos/bedroom"
@@ -129,47 +125,19 @@ input_boxes = np.array(input_boxes)
 
 print(input_boxes)
 
-# prompt SAM image predictor to get the mask for the object
-image_predictor.set_image(np.array(image.convert("RGB")))
-
 # process the detection results
 OBJECTS = class_names
 
 print(OBJECTS)
 
-# prompt SAM 2 image predictor to get the mask for the object
-masks, scores, logits = image_predictor.predict(
-    point_coords=None,
-    point_labels=None,
-    box=input_boxes,
-    multimask_output=False,
-)
-# convert the mask shape to (n, H, W)
-if masks.ndim == 4:
-    masks = masks.squeeze(1)
-
 """
 Step 3: Register each object's positive points to video predictor with seperate add_new_points call
 """
 
 assert PROMPT_TYPE_FOR_VIDEO in ["point", "box", "mask"], "SAM 2 video predictor only support point/box/mask prompt"
 
-# If you are using point prompts, we uniformly sample positive points based on the mask
-if PROMPT_TYPE_FOR_VIDEO == "point":
-    # sample the positive points from mask for each objects
-    all_sample_points = sample_points_from_masks(masks=masks, num_points=10)
-
-    for object_id, (label, points) in enumerate(zip(OBJECTS, all_sample_points), start=1):
-        labels = np.ones((points.shape[0]), dtype=np.int32)
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_points_or_box(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            points=points,
-            labels=labels,
-        )
 # Using box prompt
-elif PROMPT_TYPE_FOR_VIDEO == "box":
+if PROMPT_TYPE_FOR_VIDEO == "box":
     for object_id, (label, box) in enumerate(zip(OBJECTS, input_boxes), start=1):
         _, out_obj_ids, out_mask_logits = video_predictor.add_new_points_or_box(
             inference_state=inference_state,
@@ -177,18 +145,7 @@ elif PROMPT_TYPE_FOR_VIDEO == "box":
             obj_id=object_id,
             box=box,
         )
-# Using mask prompt is a more straightforward way
-elif PROMPT_TYPE_FOR_VIDEO == "mask":
-    for object_id, (label, mask) in enumerate(zip(OBJECTS, masks), start=1):
-        labels = np.ones((1), dtype=np.int32)
-        _, out_obj_ids, out_mask_logits = video_predictor.add_new_mask(
-            inference_state=inference_state,
-            frame_idx=ann_frame_idx,
-            obj_id=object_id,
-            mask=mask
-        )
-else:
-    raise NotImplementedError("SAM 2 video predictor only support point/box/mask prompts")
+        break
 
 """
 Step 4: Propagate the video predictor to get the segmentation results for each frame
@@ -234,4 +191,36 @@ for frame_idx, segments in video_segments.items():
 Step 6: Convert the annotated frames to video
 """
 
+def create_video_from_images(image_folder, output_video_path, frame_rate=25):
+    # define valid extension
+    valid_extensions = [".jpg", ".jpeg", ".JPG", ".JPEG", ".png", ".PNG"]
+    
+    # get all image files in the folder
+    image_files = [f for f in os.listdir(image_folder) 
+                   if os.path.splitext(f)[1] in valid_extensions]
+    image_files.sort()  # sort the files in alphabetical order
+    print(image_files)
+    if not image_files:
+        raise ValueError("No valid image files found in the specified folder.")
+    
+    # load the first image to get the dimensions of the video
+    first_image_path = os.path.join(image_folder, image_files[0])
+    first_image = cv2.imread(first_image_path)
+    height, width, _ = first_image.shape
+    
+    # create a video writer
+    fourcc = cv2.VideoWriter_fourcc(*'mp4v') # codec for saving the video
+    video_writer = cv2.VideoWriter(output_video_path, fourcc, frame_rate, (width, height))
+    
+    # write each image to the video
+    for image_file in tqdm(image_files):
+        image_path = os.path.join(image_folder, image_file)
+        image = cv2.imread(image_path)
+        video_writer.write(image)
+    
+    # source release
+    video_writer.release()
+    print(f"Video saved at {output_video_path}")
+
+
 create_video_from_images(SAVE_TRACKING_RESULTS_DIR, OUTPUT_VIDEO_PATH)

grounding_dino/.gitignore

@@ -1,146 +0,0 @@
-# IDE
-.idea/
-.vscode/
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-pip-wheel-metadata/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-.python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# vscode
-.vscode/
-output/
-outputs/
-subs/
-logs/
-
-grounding/config/configs
-grounding/version.py
-
-vis/
-tmp/

grounding_dino/Dockerfile

@@ -1,35 +0,0 @@
-FROM pytorch/pytorch:2.1.2-cuda12.1-cudnn8-runtime
-ARG DEBIAN_FRONTEND=noninteractive
-
-ENV CUDA_HOME=/usr/local/cuda \
-     TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0 7.5 8.0 8.6+PTX" \
-     SETUPTOOLS_USE_DISTUTILS=stdlib
-
-RUN conda update conda -y
-
-# Install libraries in the brand new image. 
-RUN apt-get -y update && apt-get install -y --no-install-recommends \
-         wget \
-         build-essential \
-         git \
-         python3-opencv \
-         ca-certificates && \
-    rm -rf /var/lib/apt/lists/*
-
-# Set the working directory for all the subsequent Dockerfile instructions.
-WORKDIR /opt/program
-
-RUN git clone https://github.com/IDEA-Research/GroundingDINO.git
-
-RUN mkdir weights ; cd weights ; wget -q https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth ; cd ..
-
-RUN conda install -c "nvidia/label/cuda-12.1.1" cuda -y
-ENV CUDA_HOME=$CONDA_PREFIX
-
-ENV PATH=/usr/local/cuda/bin:$PATH
-
-RUN cd GroundingDINO/ && python -m pip install .
-
-COPY docker_test.py docker_test.py
-
-CMD [ "python", "docker_test.py" ]

grounding_dino/LICENSE

@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright 2023 - present, IDEA Research.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

grounding_dino/README.md

@@ -1,370 +0,0 @@
-<div align="center">
-  <img src="./.asset/grounding_dino_logo.png" width="30%">
-</div>
-
-# :sauropod: Grounding DINO 
-
-[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/grounding-dino-marrying-dino-with-grounded/zero-shot-object-detection-on-mscoco)](https://paperswithcode.com/sota/zero-shot-object-detection-on-mscoco?p=grounding-dino-marrying-dino-with-grounded) [![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/grounding-dino-marrying-dino-with-grounded/zero-shot-object-detection-on-odinw)](https://paperswithcode.com/sota/zero-shot-object-detection-on-odinw?p=grounding-dino-marrying-dino-with-grounded) \
-[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/grounding-dino-marrying-dino-with-grounded/object-detection-on-coco-minival)](https://paperswithcode.com/sota/object-detection-on-coco-minival?p=grounding-dino-marrying-dino-with-grounded) [![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/grounding-dino-marrying-dino-with-grounded/object-detection-on-coco)](https://paperswithcode.com/sota/object-detection-on-coco?p=grounding-dino-marrying-dino-with-grounded)
-
-
-**[IDEA-CVR, IDEA-Research](https://github.com/IDEA-Research)** 
-
-[Shilong Liu](http://www.lsl.zone/), [Zhaoyang Zeng](https://scholar.google.com/citations?user=U_cvvUwAAAAJ&hl=zh-CN&oi=ao), [Tianhe Ren](https://rentainhe.github.io/), [Feng Li](https://scholar.google.com/citations?user=ybRe9GcAAAAJ&hl=zh-CN), [Hao Zhang](https://scholar.google.com/citations?user=B8hPxMQAAAAJ&hl=zh-CN), [Jie Yang](https://github.com/yangjie-cv), [Chunyuan Li](https://scholar.google.com/citations?user=Zd7WmXUAAAAJ&hl=zh-CN&oi=ao), [Jianwei Yang](https://jwyang.github.io/), [Hang Su](https://scholar.google.com/citations?hl=en&user=dxN1_X0AAAAJ&view_op=list_works&sortby=pubdate), [Jun Zhu](https://scholar.google.com/citations?hl=en&user=axsP38wAAAAJ), [Lei Zhang](https://www.leizhang.org/)<sup>:email:</sup>.
-
-
-[[`Paper`](https://arxiv.org/abs/2303.05499)] [[`Demo`](https://huggingface.co/spaces/ShilongLiu/Grounding_DINO_demo)] [[`BibTex`](#black_nib-citation)]
-
-
-PyTorch implementation and pretrained models for Grounding DINO. For details, see the paper **[Grounding DINO: Marrying DINO with Grounded Pre-Training for Open-Set Object Detection](https://arxiv.org/abs/2303.05499)**.
-
-- 🔥 **[Grounding DINO 1.5](https://github.com/IDEA-Research/Grounding-DINO-1.5-API)** is released now, which is IDEA Research's **Most Capable** Open-World Object Detection Model!
-- 🔥 **[Grounding DINO](https://arxiv.org/abs/2303.05499)** and **[Grounded SAM](https://arxiv.org/abs/2401.14159)** are now supported in Huggingface. For more convenient use, you can refer to [this documentation](https://huggingface.co/docs/transformers/model_doc/grounding-dino)
-
-## :sun_with_face: Helpful Tutorial
-
-- :grapes: [[Read our arXiv Paper](https://arxiv.org/abs/2303.05499)]
-- :apple:  [[Watch our simple introduction video on YouTube](https://youtu.be/wxWDt5UiwY8)]
-- :blossom:   &nbsp;[[Try the Colab Demo](https://colab.research.google.com/github/roboflow-ai/notebooks/blob/main/notebooks/zero-shot-object-detection-with-grounding-dino.ipynb)]
-- :sunflower: [[Try our Official Huggingface Demo](https://huggingface.co/spaces/ShilongLiu/Grounding_DINO_demo)]
-- :maple_leaf: [[Watch the Step by Step Tutorial about GroundingDINO by Roboflow AI](https://youtu.be/cMa77r3YrDk)]
-- :mushroom: [[GroundingDINO: Automated Dataset Annotation and Evaluation by Roboflow AI](https://youtu.be/C4NqaRBz_Kw)]
-- :hibiscus: [[Accelerate Image Annotation with SAM and GroundingDINO by Roboflow AI](https://youtu.be/oEQYStnF2l8)]
-- :white_flower: [[Autodistill: Train YOLOv8 with ZERO Annotations based on Grounding-DINO and Grounded-SAM by Roboflow AI](https://github.com/autodistill/autodistill)]
-
-<!-- Grounding DINO Methods | 
-[![arXiv](https://img.shields.io/badge/arXiv-2303.05499-b31b1b.svg)](https://arxiv.org/abs/2303.05499) 
-[![YouTube](https://badges.aleen42.com/src/youtube.svg)](https://youtu.be/wxWDt5UiwY8) -->
-
-<!-- Grounding DINO Demos |
-[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/roboflow-ai/notebooks/blob/main/notebooks/zero-shot-object-detection-with-grounding-dino.ipynb) -->
-<!-- [![YouTube](https://badges.aleen42.com/src/youtube.svg)](https://youtu.be/cMa77r3YrDk)
-[![HuggingFace space](https://img.shields.io/badge/🤗-HuggingFace%20Space-cyan.svg)](https://huggingface.co/spaces/ShilongLiu/Grounding_DINO_demo)
-[![YouTube](https://badges.aleen42.com/src/youtube.svg)](https://youtu.be/oEQYStnF2l8)
-[![YouTube](https://badges.aleen42.com/src/youtube.svg)](https://youtu.be/C4NqaRBz_Kw) -->
-
-## :sparkles: Highlight Projects
-
-- [Semantic-SAM: a universal image segmentation model to enable segment and recognize anything at any desired granularity.](https://github.com/UX-Decoder/Semantic-SAM), 
-- [DetGPT: Detect What You Need via Reasoning](https://github.com/OptimalScale/DetGPT)
-- [Grounded-SAM: Marrying Grounding DINO with Segment Anything](https://github.com/IDEA-Research/Grounded-Segment-Anything)
-- [Grounding DINO with Stable Diffusion](demo/image_editing_with_groundingdino_stablediffusion.ipynb)
-- [Grounding DINO with GLIGEN for Controllable Image Editing](demo/image_editing_with_groundingdino_gligen.ipynb)
-- [OpenSeeD: A Simple and Strong Openset Segmentation Model](https://github.com/IDEA-Research/OpenSeeD)
-- [SEEM: Segment Everything Everywhere All at Once](https://github.com/UX-Decoder/Segment-Everything-Everywhere-All-At-Once)
-- [X-GPT: Conversational Visual Agent supported by X-Decoder](https://github.com/microsoft/X-Decoder/tree/xgpt)
-- [GLIGEN: Open-Set Grounded Text-to-Image Generation](https://github.com/gligen/GLIGEN)
-- [LLaVA: Large Language and Vision Assistant](https://github.com/haotian-liu/LLaVA)
-
-<!-- Extensions | [Grounding DINO with Segment Anything](https://github.com/IDEA-Research/Grounded-Segment-Anything); [Grounding DINO with Stable Diffusion](demo/image_editing_with_groundingdino_stablediffusion.ipynb); [Grounding DINO with GLIGEN](demo/image_editing_with_groundingdino_gligen.ipynb)  -->
-
-
-
-<!-- Official PyTorch implementation of [Grounding DINO](https://arxiv.org/abs/2303.05499), a stronger open-set object detector. Code is available now! -->
-
-
-## :bulb: Highlight
-
-- **Open-Set Detection.** Detect **everything** with language!
-- **High Performance.** COCO zero-shot **52.5 AP** (training without COCO data!). COCO fine-tune **63.0 AP**.
-- **Flexible.** Collaboration with Stable Diffusion for Image Editting.
-
-
-
-
-## :fire: News
-- **`2023/07/18`**: We release [Semantic-SAM](https://github.com/UX-Decoder/Semantic-SAM), a universal image segmentation model to enable segment and recognize anything at any desired granularity. **Code** and **checkpoint** are available!
-- **`2023/06/17`**: We provide an example to evaluate Grounding DINO on COCO zero-shot performance.
-- **`2023/04/15`**: Refer to [CV in the Wild Readings](https://github.com/Computer-Vision-in-the-Wild/CVinW_Readings) for those who are interested in open-set recognition!
-- **`2023/04/08`**: We release [demos](demo/image_editing_with_groundingdino_gligen.ipynb) to combine [Grounding DINO](https://arxiv.org/abs/2303.05499) with [GLIGEN](https://github.com/gligen/GLIGEN)  for more controllable image editings.
-- **`2023/04/08`**: We release [demos](demo/image_editing_with_groundingdino_stablediffusion.ipynb) to combine [Grounding DINO](https://arxiv.org/abs/2303.05499) with [Stable Diffusion](https://github.com/Stability-AI/StableDiffusion) for image editings.
-- **`2023/04/06`**: We build a new demo by marrying GroundingDINO with [Segment-Anything](https://github.com/facebookresearch/segment-anything) named **[Grounded-Segment-Anything](https://github.com/IDEA-Research/Grounded-Segment-Anything)** aims to support segmentation in GroundingDINO.
-- **`2023/03/28`**: A YouTube [video](https://youtu.be/cMa77r3YrDk) about Grounding DINO and basic object detection prompt engineering. [[SkalskiP](https://github.com/SkalskiP)]
-- **`2023/03/28`**: Add a [demo](https://huggingface.co/spaces/ShilongLiu/Grounding_DINO_demo) on Hugging Face Space!
-- **`2023/03/27`**: Support CPU-only mode. Now the model can run on machines without GPUs.
-- **`2023/03/25`**: A [demo](https://colab.research.google.com/github/roboflow-ai/notebooks/blob/main/notebooks/zero-shot-object-detection-with-grounding-dino.ipynb) for Grounding DINO is available at Colab. [[SkalskiP](https://github.com/SkalskiP)]
-- **`2023/03/22`**: Code is available Now!
-
-<details open>
-<summary><font size="4">
-Description
-</font></summary>
- <a href="https://arxiv.org/abs/2303.05499">Paper</a> introduction.
-<img src=".asset/hero_figure.png" alt="ODinW" width="100%">
-Marrying <a href="https://github.com/IDEA-Research/GroundingDINO">Grounding DINO</a> and <a href="https://github.com/gligen/GLIGEN">GLIGEN</a>
-<img src="https://huggingface.co/ShilongLiu/GroundingDINO/resolve/main/GD_GLIGEN.png" alt="gd_gligen" width="100%">
-</details>
-
-## :star: Explanations/Tips for Grounding DINO Inputs and Outputs
-- Grounding DINO accepts an `(image, text)` pair as inputs.
-- It outputs `900` (by default) object boxes. Each box has similarity scores across all input words. (as shown in Figures below.)
-- We defaultly choose the boxes whose highest similarities are higher than a `box_threshold`.
-- We extract the words whose similarities are higher than the `text_threshold` as predicted labels.
-- If you want to obtain objects of specific phrases, like the `dogs` in the sentence `two dogs with a stick.`, you can select the boxes with highest text similarities with `dogs` as final outputs. 
-- Note that each word can be split to **more than one** tokens with different tokenlizers. The number of words in a sentence may not equal to the number of text tokens.
-- We suggest separating different category names with `.` for Grounding DINO.
-![model_explain1](.asset/model_explan1.PNG)
-![model_explain2](.asset/model_explan2.PNG)
-
-## :label: TODO 
-
-- [x] Release inference code and demo.
-- [x] Release checkpoints.
-- [x] Grounding DINO with Stable Diffusion and GLIGEN demos.
-- [ ] Release training codes.
-
-## :hammer_and_wrench: Install 
-
-**Note:**
-
-0. If you have a CUDA environment, please make sure the environment variable `CUDA_HOME` is set. It will be compiled under CPU-only mode if no CUDA available.
-
-Please make sure following the installation steps strictly, otherwise the program may produce: 
-```bash
-NameError: name '_C' is not defined
-```
-
-If this happened, please reinstalled the groundingDINO by reclone the git and do all the installation steps again.
- 
-#### how to check cuda:
-```bash
-echo $CUDA_HOME
-```
-If it print nothing, then it means you haven't set up the path/
-
-Run this so the environment variable will be set under current shell. 
-```bash
-export CUDA_HOME=/path/to/cuda-11.3
-```
-
-Notice the version of cuda should be aligned with your CUDA runtime, for there might exists multiple cuda at the same time. 
-
-If you want to set the CUDA_HOME permanently, store it using:
-
-```bash
-echo 'export CUDA_HOME=/path/to/cuda' >> ~/.bashrc
-```
-after that, source the bashrc file and check CUDA_HOME:
-```bash
-source ~/.bashrc
-echo $CUDA_HOME
-```
-
-In this example, /path/to/cuda-11.3 should be replaced with the path where your CUDA toolkit is installed. You can find this by typing **which nvcc** in your terminal:
-
-For instance, 
-if the output is /usr/local/cuda/bin/nvcc, then:
-```bash
-export CUDA_HOME=/usr/local/cuda
-```
-**Installation:**
-
-1.Clone the GroundingDINO repository from GitHub.
-
-```bash
-git clone https://github.com/IDEA-Research/GroundingDINO.git
-```
-
-2. Change the current directory to the GroundingDINO folder.
-
-```bash
-cd GroundingDINO/
-```
-
-3. Install the required dependencies in the current directory.
-
-```bash
-pip install -e .
-```
-
-4. Download pre-trained model weights.
-
-```bash
-mkdir weights
-cd weights
-wget -q https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth
-cd ..
-```
-
-## :arrow_forward: Demo
-Check your GPU ID (only if you're using a GPU)
-
-```bash
-nvidia-smi
-```
-Replace `{GPU ID}`, `image_you_want_to_detect.jpg`, and `"dir you want to save the output"` with appropriate values in the following command
-```bash
-CUDA_VISIBLE_DEVICES={GPU ID} python demo/inference_on_a_image.py \
--c groundingdino/config/GroundingDINO_SwinT_OGC.py \
--p weights/groundingdino_swint_ogc.pth \
--i image_you_want_to_detect.jpg \
--o "dir you want to save the output" \
--t "chair"
- [--cpu-only] # open it for cpu mode
-```
-
-If you would like to specify the phrases to detect, here is a demo:
-```bash
-CUDA_VISIBLE_DEVICES={GPU ID} python demo/inference_on_a_image.py \
--c groundingdino/config/GroundingDINO_SwinT_OGC.py \
--p ./groundingdino_swint_ogc.pth \
--i .asset/cat_dog.jpeg \
--o logs/1111 \
--t "There is a cat and a dog in the image ." \
---token_spans "[[[9, 10], [11, 14]], [[19, 20], [21, 24]]]"
- [--cpu-only] # open it for cpu mode
-```
-The token_spans specify the start and end positions of a phrases. For example, the first phrase is `[[9, 10], [11, 14]]`. `"There is a cat and a dog in the image ."[9:10] = 'a'`, `"There is a cat and a dog in the image ."[11:14] = 'cat'`. Hence it refers to the phrase `a cat` . Similarly, the `[[19, 20], [21, 24]]` refers to the phrase `a dog`.
-
-See the `demo/inference_on_a_image.py` for more details.
-
-**Running with Python:**
-
-```python
-from groundingdino.util.inference import load_model, load_image, predict, annotate
-import cv2
-
-model = load_model("groundingdino/config/GroundingDINO_SwinT_OGC.py", "weights/groundingdino_swint_ogc.pth")
-IMAGE_PATH = "weights/dog-3.jpeg"
-TEXT_PROMPT = "chair . person . dog ."
-BOX_TRESHOLD = 0.35
-TEXT_TRESHOLD = 0.25
-
-image_source, image = load_image(IMAGE_PATH)
-
-boxes, logits, phrases = predict(
-    model=model,
-    image=image,
-    caption=TEXT_PROMPT,
-    box_threshold=BOX_TRESHOLD,
-    text_threshold=TEXT_TRESHOLD
-)
-
-annotated_frame = annotate(image_source=image_source, boxes=boxes, logits=logits, phrases=phrases)
-cv2.imwrite("annotated_image.jpg", annotated_frame)
-```
-**Web UI**
-
-We also provide a demo code to integrate Grounding DINO with Gradio Web UI. See the file `demo/gradio_app.py` for more details.
-
-**Notebooks**
-
-- We release [demos](demo/image_editing_with_groundingdino_gligen.ipynb) to combine [Grounding DINO](https://arxiv.org/abs/2303.05499) with [GLIGEN](https://github.com/gligen/GLIGEN)  for more controllable image editings.
-- We release [demos](demo/image_editing_with_groundingdino_stablediffusion.ipynb) to combine [Grounding DINO](https://arxiv.org/abs/2303.05499) with [Stable Diffusion](https://github.com/Stability-AI/StableDiffusion) for image editings.
-
-## COCO Zero-shot Evaluations
-
-We provide an example to evaluate Grounding DINO zero-shot performance on COCO. The results should be **48.5**.
-
-```bash
-CUDA_VISIBLE_DEVICES=0 \
-python demo/test_ap_on_coco.py \
- -c groundingdino/config/GroundingDINO_SwinT_OGC.py \
- -p weights/groundingdino_swint_ogc.pth \
- --anno_path /path/to/annoataions/ie/instances_val2017.json \
- --image_dir /path/to/imagedir/ie/val2017
-```
-
-
-## :luggage: Checkpoints
-
-<!-- insert a table -->
-<table>
-  <thead>
-    <tr style="text-align: right;">
-      <th></th>
-      <th>name</th>
-      <th>backbone</th>
-      <th>Data</th>
-      <th>box AP on COCO</th>
-      <th>Checkpoint</th>
-      <th>Config</th>
-    </tr>
-  </thead>
-  <tbody>
-    <tr>
-      <th>1</th>
-      <td>GroundingDINO-T</td>
-      <td>Swin-T</td>
-      <td>O365,GoldG,Cap4M</td>
-      <td>48.4 (zero-shot) / 57.2 (fine-tune)</td>
-      <td><a href="https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth">GitHub link</a> | <a href="https://huggingface.co/ShilongLiu/GroundingDINO/resolve/main/groundingdino_swint_ogc.pth">HF link</a></td>
-      <td><a href="https://github.com/IDEA-Research/GroundingDINO/blob/main/groundingdino/config/GroundingDINO_SwinT_OGC.py">link</a></td>
-    </tr>
-    <tr>
-      <th>2</th>
-      <td>GroundingDINO-B</td>
-      <td>Swin-B</td>
-      <td>COCO,O365,GoldG,Cap4M,OpenImage,ODinW-35,RefCOCO</td>
-      <td>56.7 </td>
-      <td><a href="https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha2/groundingdino_swinb_cogcoor.pth">GitHub link</a>  | <a href="https://huggingface.co/ShilongLiu/GroundingDINO/resolve/main/groundingdino_swinb_cogcoor.pth">HF link</a> 
-      <td><a href="https://github.com/IDEA-Research/GroundingDINO/blob/main/groundingdino/config/GroundingDINO_SwinB_cfg.py">link</a></td>
-    </tr>
-  </tbody>
-</table>
-
-## :medal_military: Results
-
-<details open>
-<summary><font size="4">
-COCO Object Detection Results
-</font></summary>
-<img src=".asset/COCO.png" alt="COCO" width="100%">
-</details>
-
-<details open>
-<summary><font size="4">
-ODinW Object Detection Results
-</font></summary>
-<img src=".asset/ODinW.png" alt="ODinW" width="100%">
-</details>
-
-<details open>
-<summary><font size="4">
-Marrying Grounding DINO with <a href="https://github.com/Stability-AI/StableDiffusion">Stable Diffusion</a> for Image Editing
-</font></summary>
-See our example <a href="https://github.com/IDEA-Research/GroundingDINO/blob/main/demo/image_editing_with_groundingdino_stablediffusion.ipynb">notebook</a> for more details.
-<img src=".asset/GD_SD.png" alt="GD_SD" width="100%">
-</details>
-
-
-<details open>
-<summary><font size="4">
-Marrying Grounding DINO with <a href="https://github.com/gligen/GLIGEN">GLIGEN</a> for more Detailed Image Editing.
-</font></summary>
-See our example <a href="https://github.com/IDEA-Research/GroundingDINO/blob/main/demo/image_editing_with_groundingdino_gligen.ipynb">notebook</a> for more details.
-<img src=".asset/GD_GLIGEN.png" alt="GD_GLIGEN" width="100%">
-</details>
-
-## :sauropod: Model: Grounding DINO
-
-Includes: a text backbone, an image backbone, a feature enhancer, a language-guided query selection, and a cross-modality decoder.
-
-![arch](.asset/arch.png)
-
-
-## :hearts: Acknowledgement
-
-Our model is related to [DINO](https://github.com/IDEA-Research/DINO) and [GLIP](https://github.com/microsoft/GLIP). Thanks for their great work!
-
-We also thank great previous work including DETR, Deformable DETR, SMCA, Conditional DETR, Anchor DETR, Dynamic DETR, DAB-DETR, DN-DETR, etc. More related work are available at [Awesome Detection Transformer](https://github.com/IDEACVR/awesome-detection-transformer). A new toolbox [detrex](https://github.com/IDEA-Research/detrex) is available as well.
-
-Thanks [Stable Diffusion](https://github.com/Stability-AI/StableDiffusion) and [GLIGEN](https://github.com/gligen/GLIGEN) for their awesome models.
-
-
-## :black_nib: Citation
-
-If you find our work helpful for your research, please consider citing the following BibTeX entry.   
-
-```bibtex
-@article{liu2023grounding,
-  title={Grounding dino: Marrying dino with grounded pre-training for open-set object detection},
-  author={Liu, Shilong and Zeng, Zhaoyang and Ren, Tianhe and Li, Feng and Zhang, Hao and Yang, Jie and Li, Chunyuan and Yang, Jianwei and Su, Hang and Zhu, Jun and others},
-  journal={arXiv preprint arXiv:2303.05499},
-  year={2023}
-}
-```
-
-
-
-

grounding_dino/demo/create_coco_dataset.py

@@ -1,83 +0,0 @@
-import typer
-from groundingdino.util.inference import load_model, load_image, predict
-from tqdm import tqdm
-import torchvision
-import torch
-import fiftyone as fo
-
-
-def main(
-        image_directory: str = 'test_grounding_dino',
-        text_prompt: str = 'bus, car',
-        box_threshold: float = 0.15, 
-        text_threshold: float = 0.10,
-        export_dataset: bool = False,
-        view_dataset: bool = False,
-        export_annotated_images: bool = True,
-        weights_path : str = "groundingdino_swint_ogc.pth",
-        config_path: str = "../../GroundingDINO/groundingdino/config/GroundingDINO_SwinT_OGC.py",
-        subsample: int = None,
-    ):
-
-    model = load_model(config_path, weights_path)
-    
-    dataset = fo.Dataset.from_images_dir(image_directory)
-
-    samples = []
-
-    if subsample is not None: 
-        
-        if subsample < len(dataset):
-            dataset = dataset.take(subsample).clone()
-    
-    for sample in tqdm(dataset):
-
-        image_source, image = load_image(sample.filepath)
-
-        boxes, logits, phrases = predict(
-            model=model, 
-            image=image, 
-            caption=text_prompt, 
-            box_threshold=box_threshold, 
-            text_threshold=text_threshold,
-        )
-
-        detections = [] 
-
-        for box, logit, phrase in zip(boxes, logits, phrases):
-
-            rel_box = torchvision.ops.box_convert(box, 'cxcywh', 'xywh')
-
-            detections.append(
-                fo.Detection(
-                    label=phrase, 
-                    bounding_box=rel_box,
-                    confidence=logit,
-            ))
-
-        # Store detections in a field name of your choice
-        sample["detections"] = fo.Detections(detections=detections)
-        sample.save()
-
-    # loads the voxel fiftyone UI ready for viewing the dataset.
-    if view_dataset:
-        session = fo.launch_app(dataset)
-        session.wait()
-        
-    # exports COCO dataset ready for training
-    if export_dataset:
-        dataset.export(
-            'coco_dataset',
-            dataset_type=fo.types.COCODetectionDataset,
-        )
-        
-    # saves bounding boxes plotted on the input images to disk
-    if export_annotated_images:
-        dataset.draw_labels(
-            'images_with_bounding_boxes',
-            label_fields=['detections']
-        )
-
-
-if __name__ == '__main__':
-    typer.run(main)

grounding_dino/demo/gradio_app.py

@@ -1,125 +0,0 @@
-import argparse
-from functools import partial
-import cv2
-import requests
-import os
-from io import BytesIO
-from PIL import Image
-import numpy as np
-from pathlib import Path
-
-
-import warnings
-
-import torch
-
-# prepare the environment
-os.system("python setup.py build develop --user")
-os.system("pip install packaging==21.3")
-os.system("pip install gradio==3.50.2")
-
-
-warnings.filterwarnings("ignore")
-
-import gradio as gr
-
-from groundingdino.models import build_model
-from groundingdino.util.slconfig import SLConfig
-from groundingdino.util.utils import clean_state_dict
-from groundingdino.util.inference import annotate, load_image, predict
-import groundingdino.datasets.transforms as T
-
-from huggingface_hub import hf_hub_download
-
-
-
-# Use this command for evaluate the Grounding DINO model
-config_file = "groundingdino/config/GroundingDINO_SwinT_OGC.py"
-ckpt_repo_id = "ShilongLiu/GroundingDINO"
-ckpt_filenmae = "groundingdino_swint_ogc.pth"
-
-
-def load_model_hf(model_config_path, repo_id, filename, device='cpu'):
-    args = SLConfig.fromfile(model_config_path) 
-    model = build_model(args)
-    args.device = device
-
-    cache_file = hf_hub_download(repo_id=repo_id, filename=filename)
-    checkpoint = torch.load(cache_file, map_location='cpu')
-    log = model.load_state_dict(clean_state_dict(checkpoint['model']), strict=False)
-    print("Model loaded from {} \n => {}".format(cache_file, log))
-    _ = model.eval()
-    return model    
-
-def image_transform_grounding(init_image):
-    transform = T.Compose([
-        T.RandomResize([800], max_size=1333),
-        T.ToTensor(),
-        T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
-    ])
-    image, _ = transform(init_image, None) # 3, h, w
-    return init_image, image
-
-def image_transform_grounding_for_vis(init_image):
-    transform = T.Compose([
-        T.RandomResize([800], max_size=1333),
-    ])
-    image, _ = transform(init_image, None) # 3, h, w
-    return image
-
-model = load_model_hf(config_file, ckpt_repo_id, ckpt_filenmae)
-
-def run_grounding(input_image, grounding_caption, box_threshold, text_threshold):
-    init_image = input_image.convert("RGB")
-    original_size = init_image.size
-
-    _, image_tensor = image_transform_grounding(init_image)
-    image_pil: Image = image_transform_grounding_for_vis(init_image)
-
-    # run grounidng
-    boxes, logits, phrases = predict(model, image_tensor, grounding_caption, box_threshold, text_threshold, device='cpu')
-    annotated_frame = annotate(image_source=np.asarray(image_pil), boxes=boxes, logits=logits, phrases=phrases)
-    image_with_box = Image.fromarray(cv2.cvtColor(annotated_frame, cv2.COLOR_BGR2RGB))
-
-
-    return image_with_box
-
-if __name__ == "__main__":
-
-    parser = argparse.ArgumentParser("Grounding DINO demo", add_help=True)
-    parser.add_argument("--debug", action="store_true", help="using debug mode")
-    parser.add_argument("--share", action="store_true", help="share the app")
-    args = parser.parse_args()
-
-    block = gr.Blocks().queue()
-    with block:
-        gr.Markdown("# [Grounding DINO](https://github.com/IDEA-Research/GroundingDINO)")
-        gr.Markdown("### Open-World Detection with Grounding DINO")
-
-        with gr.Row():
-            with gr.Column():
-                input_image = gr.Image(source='upload', type="pil")
-                grounding_caption = gr.Textbox(label="Detection Prompt")
-                run_button = gr.Button(label="Run")
-                with gr.Accordion("Advanced options", open=False):
-                    box_threshold = gr.Slider(
-                        label="Box Threshold", minimum=0.0, maximum=1.0, value=0.25, step=0.001
-                    )
-                    text_threshold = gr.Slider(
-                        label="Text Threshold", minimum=0.0, maximum=1.0, value=0.25, step=0.001
-                    )
-
-            with gr.Column():
-                gallery = gr.outputs.Image(
-                    type="pil",
-                    # label="grounding results"
-                ).style(full_width=True, full_height=True)
-                # gallery = gr.Gallery(label="Generated images", show_label=False).style(
-                #         grid=[1], height="auto", container=True, full_width=True, full_height=True)
-
-        run_button.click(fn=run_grounding, inputs=[
-                        input_image, grounding_caption, box_threshold, text_threshold], outputs=[gallery])
-
-
-    block.launch(server_name='0.0.0.0', server_port=7579, debug=args.debug, share=args.share)
-

grounding_dino/demo/inference_on_a_image.py

@@ -1,214 +0,0 @@
-import argparse
-import os
-import sys
-
-import numpy as np
-import torch
-from PIL import Image, ImageDraw, ImageFont
-
-import groundingdino.datasets.transforms as T
-from groundingdino.models import build_model
-from groundingdino.util import box_ops
-from groundingdino.util.slconfig import SLConfig
-from groundingdino.util.utils import clean_state_dict, get_phrases_from_posmap
-from groundingdino.util.vl_utils import create_positive_map_from_span
-
-
-def plot_boxes_to_image(image_pil, tgt):
-    H, W = tgt["size"]
-    boxes = tgt["boxes"]
-    labels = tgt["labels"]
-    assert len(boxes) == len(labels), "boxes and labels must have same length"
-
-    draw = ImageDraw.Draw(image_pil)
-    mask = Image.new("L", image_pil.size, 0)
-    mask_draw = ImageDraw.Draw(mask)
-
-    # draw boxes and masks
-    for box, label in zip(boxes, labels):
-        # from 0..1 to 0..W, 0..H
-        box = box * torch.Tensor([W, H, W, H])
-        # from xywh to xyxy
-        box[:2] -= box[2:] / 2
-        box[2:] += box[:2]
-        # random color
-        color = tuple(np.random.randint(0, 255, size=3).tolist())
-        # draw
-        x0, y0, x1, y1 = box
-        x0, y0, x1, y1 = int(x0), int(y0), int(x1), int(y1)
-
-        draw.rectangle([x0, y0, x1, y1], outline=color, width=6)
-        # draw.text((x0, y0), str(label), fill=color)
-
-        font = ImageFont.load_default()
-        if hasattr(font, "getbbox"):
-            bbox = draw.textbbox((x0, y0), str(label), font)
-        else:
-            w, h = draw.textsize(str(label), font)
-            bbox = (x0, y0, w + x0, y0 + h)
-        # bbox = draw.textbbox((x0, y0), str(label))
-        draw.rectangle(bbox, fill=color)
-        draw.text((x0, y0), str(label), fill="white")
-
-        mask_draw.rectangle([x0, y0, x1, y1], fill=255, width=6)
-
-    return image_pil, mask
-
-
-def load_image(image_path):
-    # load image
-    image_pil = Image.open(image_path).convert("RGB")  # load image
-
-    transform = T.Compose(
-        [
-            T.RandomResize([800], max_size=1333),
-            T.ToTensor(),
-            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
-        ]
-    )
-    image, _ = transform(image_pil, None)  # 3, h, w
-    return image_pil, image
-
-
-def load_model(model_config_path, model_checkpoint_path, cpu_only=False):
-    args = SLConfig.fromfile(model_config_path)
-    args.device = "cuda" if not cpu_only else "cpu"
-    model = build_model(args)
-    checkpoint = torch.load(model_checkpoint_path, map_location="cpu")
-    load_res = model.load_state_dict(clean_state_dict(checkpoint["model"]), strict=False)
-    print(load_res)
-    _ = model.eval()
-    return model
-
-
-def get_grounding_output(model, image, caption, box_threshold, text_threshold=None, with_logits=True, cpu_only=False, token_spans=None):
-    assert text_threshold is not None or token_spans is not None, "text_threshould and token_spans should not be None at the same time!"
-    caption = caption.lower()
-    caption = caption.strip()
-    if not caption.endswith("."):
-        caption = caption + "."
-    device = "cuda" if not cpu_only else "cpu"
-    model = model.to(device)
-    image = image.to(device)
-    with torch.no_grad():
-        outputs = model(image[None], captions=[caption])
-    logits = outputs["pred_logits"].sigmoid()[0]  # (nq, 256)
-    boxes = outputs["pred_boxes"][0]  # (nq, 4)
-
-    # filter output
-    if token_spans is None:
-        logits_filt = logits.cpu().clone()
-        boxes_filt = boxes.cpu().clone()
-        filt_mask = logits_filt.max(dim=1)[0] > box_threshold
-        logits_filt = logits_filt[filt_mask]  # num_filt, 256
-        boxes_filt = boxes_filt[filt_mask]  # num_filt, 4
-
-        # get phrase
-        tokenlizer = model.tokenizer
-        tokenized = tokenlizer(caption)
-        # build pred
-        pred_phrases = []
-        for logit, box in zip(logits_filt, boxes_filt):
-            pred_phrase = get_phrases_from_posmap(logit > text_threshold, tokenized, tokenlizer)
-            if with_logits:
-                pred_phrases.append(pred_phrase + f"({str(logit.max().item())[:4]})")
-            else:
-                pred_phrases.append(pred_phrase)
-    else:
-        # given-phrase mode
-        positive_maps = create_positive_map_from_span(
-            model.tokenizer(text_prompt),
-            token_span=token_spans
-        ).to(image.device) # n_phrase, 256
-
-        logits_for_phrases = positive_maps @ logits.T # n_phrase, nq
-        all_logits = []
-        all_phrases = []
-        all_boxes = []
-        for (token_span, logit_phr) in zip(token_spans, logits_for_phrases):
-            # get phrase
-            phrase = ' '.join([caption[_s:_e] for (_s, _e) in token_span])
-            # get mask
-            filt_mask = logit_phr > box_threshold
-            # filt box
-            all_boxes.append(boxes[filt_mask])
-            # filt logits
-            all_logits.append(logit_phr[filt_mask])
-            if with_logits:
-                logit_phr_num = logit_phr[filt_mask]
-                all_phrases.extend([phrase + f"({str(logit.item())[:4]})" for logit in logit_phr_num])
-            else:
-                all_phrases.extend([phrase for _ in range(len(filt_mask))])
-        boxes_filt = torch.cat(all_boxes, dim=0).cpu()
-        pred_phrases = all_phrases
-
-
-    return boxes_filt, pred_phrases
-
-
-if __name__ == "__main__":
-
-    parser = argparse.ArgumentParser("Grounding DINO example", add_help=True)
-    parser.add_argument("--config_file", "-c", type=str, required=True, help="path to config file")
-    parser.add_argument(
-        "--checkpoint_path", "-p", type=str, required=True, help="path to checkpoint file"
-    )
-    parser.add_argument("--image_path", "-i", type=str, required=True, help="path to image file")
-    parser.add_argument("--text_prompt", "-t", type=str, required=True, help="text prompt")
-    parser.add_argument(
-        "--output_dir", "-o", type=str, default="outputs", required=True, help="output directory"
-    )
-
-    parser.add_argument("--box_threshold", type=float, default=0.3, help="box threshold")
-    parser.add_argument("--text_threshold", type=float, default=0.25, help="text threshold")
-    parser.add_argument("--token_spans", type=str, default=None, help=
-                        "The positions of start and end positions of phrases of interest. \
-                        For example, a caption is 'a cat and a dog', \
-                        if you would like to detect 'cat', the token_spans should be '[[[2, 5]], ]', since 'a cat and a dog'[2:5] is 'cat'. \
-                        if you would like to detect 'a cat', the token_spans should be '[[[0, 1], [2, 5]], ]', since 'a cat and a dog'[0:1] is 'a', and 'a cat and a dog'[2:5] is 'cat'. \
-                        ")
-
-    parser.add_argument("--cpu-only", action="store_true", help="running on cpu only!, default=False")
-    args = parser.parse_args()
-
-    # cfg
-    config_file = args.config_file  # change the path of the model config file
-    checkpoint_path = args.checkpoint_path  # change the path of the model
-    image_path = args.image_path
-    text_prompt = args.text_prompt
-    output_dir = args.output_dir
-    box_threshold = args.box_threshold
-    text_threshold = args.text_threshold
-    token_spans = args.token_spans
-
-    # make dir
-    os.makedirs(output_dir, exist_ok=True)
-    # load image
-    image_pil, image = load_image(image_path)
-    # load model
-    model = load_model(config_file, checkpoint_path, cpu_only=args.cpu_only)
-
-    # visualize raw image
-    image_pil.save(os.path.join(output_dir, "raw_image.jpg"))
-
-    # set the text_threshold to None if token_spans is set.
-    if token_spans is not None:
-        text_threshold = None
-        print("Using token_spans. Set the text_threshold to None.")
-
-
-    # run model
-    boxes_filt, pred_phrases = get_grounding_output(
-        model, image, text_prompt, box_threshold, text_threshold, cpu_only=args.cpu_only, token_spans=eval(f"{token_spans}")
-    )
-
-    # visualize pred
-    size = image_pil.size
-    pred_dict = {
-        "boxes": boxes_filt,
-        "size": [size[1], size[0]],  # H,W
-        "labels": pred_phrases,
-    }
-    # import ipdb; ipdb.set_trace()
-    image_with_box = plot_boxes_to_image(image_pil, pred_dict)[0]
-    image_with_box.save(os.path.join(output_dir, "pred.jpg"))

grounding_dino/demo/test_ap_on_coco.py

@@ -1,233 +0,0 @@
-import argparse
-import os
-import sys
-import time
-
-import numpy as np
-import torch
-import torch.nn as nn
-from torch.utils.data import DataLoader, DistributedSampler
-
-from groundingdino.models import build_model
-import groundingdino.datasets.transforms as T
-from groundingdino.util import box_ops, get_tokenlizer
-from groundingdino.util.misc import clean_state_dict, collate_fn
-from groundingdino.util.slconfig import SLConfig
-
-# from torchvision.datasets import CocoDetection
-import torchvision
-
-from groundingdino.util.vl_utils import build_captions_and_token_span, create_positive_map_from_span
-from groundingdino.datasets.cocogrounding_eval import CocoGroundingEvaluator
-
-
-def load_model(model_config_path: str, model_checkpoint_path: str, device: str = "cuda"):
-    args = SLConfig.fromfile(model_config_path)
-    args.device = device
-    model = build_model(args)
-    checkpoint = torch.load(model_checkpoint_path, map_location="cpu")
-    model.load_state_dict(clean_state_dict(checkpoint["model"]), strict=False)
-    model.eval()
-    return model
-
-
-class CocoDetection(torchvision.datasets.CocoDetection):
-    def __init__(self, img_folder, ann_file, transforms):
-        super().__init__(img_folder, ann_file)
-        self._transforms = transforms
-
-    def __getitem__(self, idx):
-        img, target = super().__getitem__(idx)  # target: list
-
-        # import ipdb; ipdb.set_trace()
-
-        w, h = img.size
-        boxes = [obj["bbox"] for obj in target]
-        boxes = torch.as_tensor(boxes, dtype=torch.float32).reshape(-1, 4)
-        boxes[:, 2:] += boxes[:, :2]  # xywh -> xyxy
-        boxes[:, 0::2].clamp_(min=0, max=w)
-        boxes[:, 1::2].clamp_(min=0, max=h)
-        # filt invalid boxes/masks/keypoints
-        keep = (boxes[:, 3] > boxes[:, 1]) & (boxes[:, 2] > boxes[:, 0])
-        boxes = boxes[keep]
-
-        target_new = {}
-        image_id = self.ids[idx]
-        target_new["image_id"] = image_id
-        target_new["boxes"] = boxes
-        target_new["orig_size"] = torch.as_tensor([int(h), int(w)])
-
-        if self._transforms is not None:
-            img, target = self._transforms(img, target_new)
-
-        return img, target
-
-
-class PostProcessCocoGrounding(nn.Module):
-    """ This module converts the model's output into the format expected by the coco api"""
-
-    def __init__(self, num_select=300, coco_api=None, tokenlizer=None) -> None:
-        super().__init__()
-        self.num_select = num_select
-
-        assert coco_api is not None
-        category_dict = coco_api.dataset['categories']
-        cat_list = [item['name'] for item in category_dict]
-        captions, cat2tokenspan = build_captions_and_token_span(cat_list, True)
-        tokenspanlist = [cat2tokenspan[cat] for cat in cat_list]
-        positive_map = create_positive_map_from_span(
-            tokenlizer(captions), tokenspanlist)  # 80, 256. normed
-
-        id_map = {0: 1, 1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: 7, 7: 8, 8: 9, 9: 10, 10: 11, 11: 13, 12: 14, 13: 15, 14: 16, 15: 17, 16: 18, 17: 19, 18: 20, 19: 21, 20: 22, 21: 23, 22: 24, 23: 25, 24: 27, 25: 28, 26: 31, 27: 32, 28: 33, 29: 34, 30: 35, 31: 36, 32: 37, 33: 38, 34: 39, 35: 40, 36: 41, 37: 42, 38: 43, 39: 44, 40: 46,
-                  41: 47, 42: 48, 43: 49, 44: 50, 45: 51, 46: 52, 47: 53, 48: 54, 49: 55, 50: 56, 51: 57, 52: 58, 53: 59, 54: 60, 55: 61, 56: 62, 57: 63, 58: 64, 59: 65, 60: 67, 61: 70, 62: 72, 63: 73, 64: 74, 65: 75, 66: 76, 67: 77, 68: 78, 69: 79, 70: 80, 71: 81, 72: 82, 73: 84, 74: 85, 75: 86, 76: 87, 77: 88, 78: 89, 79: 90}
-
-        # build a mapping from label_id to pos_map
-        new_pos_map = torch.zeros((91, 256))
-        for k, v in id_map.items():
-            new_pos_map[v] = positive_map[k]
-        self.positive_map = new_pos_map
-
-    @torch.no_grad()
-    def forward(self, outputs, target_sizes, not_to_xyxy=False):
-        """ Perform the computation
-        Parameters:
-            outputs: raw outputs of the model
-            target_sizes: tensor of dimension [batch_size x 2] containing the size of each images of the batch
-                          For evaluation, this must be the original image size (before any data augmentation)
-                          For visualization, this should be the image size after data augment, but before padding
-        """
-        num_select = self.num_select
-        out_logits, out_bbox = outputs['pred_logits'], outputs['pred_boxes']
-
-        # pos map to logit
-        prob_to_token = out_logits.sigmoid()  # bs, 100, 256
-        pos_maps = self.positive_map.to(prob_to_token.device)
-        # (bs, 100, 256) @ (91, 256).T -> (bs, 100, 91)
-        prob_to_label = prob_to_token @ pos_maps.T
-
-        # if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO':
-        #     import ipdb; ipdb.set_trace()
-
-        assert len(out_logits) == len(target_sizes)
-        assert target_sizes.shape[1] == 2
-
-        prob = prob_to_label
-        topk_values, topk_indexes = torch.topk(
-            prob.view(out_logits.shape[0], -1), num_select, dim=1)
-        scores = topk_values
-        topk_boxes = topk_indexes // prob.shape[2]
-        labels = topk_indexes % prob.shape[2]
-
-        if not_to_xyxy:
-            boxes = out_bbox
-        else:
-            boxes = box_ops.box_cxcywh_to_xyxy(out_bbox)
-
-        boxes = torch.gather(
-            boxes, 1, topk_boxes.unsqueeze(-1).repeat(1, 1, 4))
-
-        # and from relative [0, 1] to absolute [0, height] coordinates
-        img_h, img_w = target_sizes.unbind(1)
-        scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1)
-        boxes = boxes * scale_fct[:, None, :]
-
-        results = [{'scores': s, 'labels': l, 'boxes': b}
-                   for s, l, b in zip(scores, labels, boxes)]
-
-        return results
-
-
-def main(args):
-    # config
-    cfg = SLConfig.fromfile(args.config_file)
-
-    # build model
-    model = load_model(args.config_file, args.checkpoint_path)
-    model = model.to(args.device)
-    model = model.eval()
-
-    # build dataloader
-    transform = T.Compose(
-        [
-            T.RandomResize([800], max_size=1333),
-            T.ToTensor(),
-            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
-        ]
-    )
-    dataset = CocoDetection(
-        args.image_dir, args.anno_path, transforms=transform)
-    data_loader = DataLoader(
-        dataset, batch_size=1, shuffle=False, num_workers=args.num_workers, collate_fn=collate_fn)
-
-    # build post processor
-    tokenlizer = get_tokenlizer.get_tokenlizer(cfg.text_encoder_type)
-    postprocessor = PostProcessCocoGrounding(
-        coco_api=dataset.coco, tokenlizer=tokenlizer)
-
-    # build evaluator
-    evaluator = CocoGroundingEvaluator(
-        dataset.coco, iou_types=("bbox",), useCats=True)
-
-    # build captions
-    category_dict = dataset.coco.dataset['categories']
-    cat_list = [item['name'] for item in category_dict]
-    caption = " . ".join(cat_list) + ' .'
-    print("Input text prompt:", caption)
-
-    # run inference
-    start = time.time()
-    for i, (images, targets) in enumerate(data_loader):
-        # get images and captions
-        images = images.tensors.to(args.device)
-        bs = images.shape[0]
-        input_captions = [caption] * bs
-
-        # feed to the model
-        outputs = model(images, captions=input_captions)
-
-        orig_target_sizes = torch.stack(
-            [t["orig_size"] for t in targets], dim=0).to(images.device)
-        results = postprocessor(outputs, orig_target_sizes)
-        cocogrounding_res = {
-            target["image_id"]: output for target, output in zip(targets, results)}
-        evaluator.update(cocogrounding_res)
-
-        if (i+1) % 30 == 0:
-            used_time = time.time() - start
-            eta = len(data_loader) / (i+1e-5) * used_time - used_time
-            print(
-                f"processed {i}/{len(data_loader)} images. time: {used_time:.2f}s, ETA: {eta:.2f}s")
-
-    evaluator.synchronize_between_processes()
-    evaluator.accumulate()
-    evaluator.summarize()
-
-    print("Final results:", evaluator.coco_eval["bbox"].stats.tolist())
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        "Grounding DINO eval on COCO", add_help=True)
-    # load model
-    parser.add_argument("--config_file", "-c", type=str,
-                        required=True, help="path to config file")
-    parser.add_argument(
-        "--checkpoint_path", "-p", type=str, required=True, help="path to checkpoint file"
-    )
-    parser.add_argument("--device", type=str, default="cuda",
-                        help="running device (default: cuda)")
-
-    # post processing
-    parser.add_argument("--num_select", type=int, default=300,
-                        help="number of topk to select")
-
-    # coco info
-    parser.add_argument("--anno_path", type=str,
-                        required=True, help="coco root")
-    parser.add_argument("--image_dir", type=str,
-                        required=True, help="coco image dir")
-    parser.add_argument("--num_workers", type=int, default=4,
-                        help="number of workers for dataloader")
-    args = parser.parse_args()
-
-    main(args)

grounding_dino/docker_test.py

@@ -1,8 +0,0 @@
-from groundingdino.util.inference import load_model, load_image, predict, annotate
-import torch
-import cv2
-
-model = load_model("groundingdino/config/GroundingDINO_SwinT_OGC.pyy", "weights/groundingdino_swint_ogc.pth")
-model = model.to('cuda:0')
-print(torch.cuda.is_available())
-print('DONE!')

grounding_dino/environment.yaml

@@ -1,248 +0,0 @@
-name: dino
-channels:
-  - pytorch
-  - nvidia
-  - conda-forge
-  - defaults
-dependencies:
-  - addict=2.4.0=pyhd8ed1ab_2
-  - aiohttp=3.8.5=py39ha55989b_0
-  - aiosignal=1.3.1=pyhd8ed1ab_0
-  - asttokens=2.0.5=pyhd3eb1b0_0
-  - async-timeout=4.0.3=pyhd8ed1ab_0
-  - attrs=23.1.0=pyh71513ae_1
-  - aws-c-auth=0.7.0=h6f3c987_2
-  - aws-c-cal=0.6.0=h6ba3258_0
-  - aws-c-common=0.8.23=hcfcfb64_0
-  - aws-c-compression=0.2.17=h420beca_1
-  - aws-c-event-stream=0.3.1=had47b81_1
-  - aws-c-http=0.7.11=h72ba615_0
-  - aws-c-io=0.13.28=ha35c040_0
-  - aws-c-mqtt=0.8.14=h4941efa_2
-  - aws-c-s3=0.3.13=he04eaa7_2
-  - aws-c-sdkutils=0.1.11=h420beca_1
-  - aws-checksums=0.1.16=h420beca_1
-  - aws-crt-cpp=0.20.3=h247a981_4
-  - aws-sdk-cpp=1.10.57=h1a0519f_17
-  - backcall=0.2.0=pyhd3eb1b0_0
-  - blas=2.118=mkl
-  - blas-devel=3.9.0=18_win64_mkl
-  - brotli=1.0.9=hcfcfb64_9
-  - brotli-bin=1.0.9=hcfcfb64_9
-  - brotli-python=1.0.9=py39h99910a6_9
-  - bzip2=1.0.8=h8ffe710_4
-  - c-ares=1.19.1=hcfcfb64_0
-  - ca-certificates=2023.08.22=haa95532_0
-  - certifi=2023.7.22=py39haa95532_0
-  - charset-normalizer=3.2.0=pyhd8ed1ab_0
-  - click=8.1.7=win_pyh7428d3b_0
-  - colorama=0.4.6=pyhd8ed1ab_0
-  - comm=0.1.2=py39haa95532_0
-  - contourpy=1.1.1=py39h1f6ef14_1
-  - cuda-cccl=12.2.140=0
-  - cuda-cudart=11.8.89=0
-  - cuda-cudart-dev=11.8.89=0
-  - cuda-cupti=11.8.87=0
-  - cuda-libraries=11.8.0=0
-  - cuda-libraries-dev=11.8.0=0
-  - cuda-nvrtc=11.8.89=0
-  - cuda-nvrtc-dev=11.8.89=0
-  - cuda-nvtx=11.8.86=0
-  - cuda-profiler-api=12.2.140=0
-  - cuda-runtime=11.8.0=0
-  - cycler=0.11.0=pyhd8ed1ab_0
-  - cython=3.0.0=py39h2bbff1b_0
-  - dataclasses=0.8=pyhc8e2a94_3
-  - datasets=2.14.5=pyhd8ed1ab_0
-  - debugpy=1.6.7=py39hd77b12b_0
-  - decorator=5.1.1=pyhd3eb1b0_0
-  - dill=0.3.7=pyhd8ed1ab_0
-  - exceptiongroup=1.0.4=py39haa95532_0
-  - executing=0.8.3=pyhd3eb1b0_0
-  - filelock=3.12.4=pyhd8ed1ab_0
-  - fonttools=4.42.1=py39ha55989b_0
-  - freeglut=3.2.2=h63175ca_2
-  - freetype=2.12.1=hdaf720e_2
-  - frozenlist=1.4.0=py39ha55989b_1
-  - fsspec=2023.6.0=pyh1a96a4e_0
-  - gettext=0.21.1=h5728263_0
-  - glib=2.78.0=h12be248_0
-  - glib-tools=2.78.0=h12be248_0
-  - gst-plugins-base=1.22.6=h001b923_1
-  - gstreamer=1.22.6=hb4038d2_1
-  - huggingface_hub=0.17.3=pyhd8ed1ab_0
-  - icu=70.1=h0e60522_0
-  - idna=3.4=pyhd8ed1ab_0
-  - importlib-metadata=6.8.0=pyha770c72_0
-  - importlib-resources=6.1.0=pyhd8ed1ab_0
-  - importlib_metadata=6.8.0=hd8ed1ab_0
-  - importlib_resources=6.1.0=pyhd8ed1ab_0
-  - intel-openmp=2023.2.0=h57928b3_49503
-  - ipykernel=6.25.0=py39h9909e9c_0
-  - ipython=8.15.0=py39haa95532_0
-  - jasper=2.0.33=hc2e4405_1
-  - jedi=0.18.1=py39haa95532_1
-  - jinja2=3.1.2=pyhd8ed1ab_1
-  - joblib=1.3.2=pyhd8ed1ab_0
-  - jpeg=9e=hcfcfb64_3
-  - jupyter_client=8.1.0=py39haa95532_0
-  - jupyter_core=5.3.0=py39haa95532_0
-  - kiwisolver=1.4.5=py39h1f6ef14_1
-  - krb5=1.20.1=heb0366b_0
-  - lcms2=2.14=h90d422f_0
-  - lerc=4.0.0=h63175ca_0
-  - libabseil=20230125.3=cxx17_h63175ca_0
-  - libarrow=12.0.1=h12e5d06_5_cpu
-  - libblas=3.9.0=18_win64_mkl
-  - libbrotlicommon=1.0.9=hcfcfb64_9
-  - libbrotlidec=1.0.9=hcfcfb64_9
-  - libbrotlienc=1.0.9=hcfcfb64_9
-  - libcblas=3.9.0=18_win64_mkl
-  - libclang=15.0.7=default_h77d9078_3
-  - libclang13=15.0.7=default_h77d9078_3
-  - libcrc32c=1.1.2=h0e60522_0
-  - libcublas=11.11.3.6=0
-  - libcublas-dev=11.11.3.6=0
-  - libcufft=10.9.0.58=0
-  - libcufft-dev=10.9.0.58=0
-  - libcurand=10.3.3.141=0
-  - libcurand-dev=10.3.3.141=0
-  - libcurl=8.1.2=h68f0423_0
-  - libcusolver=11.4.1.48=0
-  - libcusolver-dev=11.4.1.48=0
-  - libcusparse=11.7.5.86=0
-  - libcusparse-dev=11.7.5.86=0
-  - libdeflate=1.14=hcfcfb64_0
-  - libevent=2.1.12=h3671451_1
-  - libffi=3.4.2=h8ffe710_5
-  - libglib=2.78.0=he8f3873_0
-  - libgoogle-cloud=2.12.0=h00b2bdc_1
-  - libgrpc=1.54.3=ha177ca7_0
-  - libhwloc=2.9.3=default_haede6df_1009
-  - libiconv=1.17=h8ffe710_0
-  - liblapack=3.9.0=18_win64_mkl
-  - liblapacke=3.9.0=18_win64_mkl
-  - libnpp=11.8.0.86=0
-  - libnpp-dev=11.8.0.86=0
-  - libnvjpeg=11.9.0.86=0
-  - libnvjpeg-dev=11.9.0.86=0
-  - libogg=1.3.4=h8ffe710_1
-  - libopencv=4.5.3=py39h488c12c_8
-  - libpng=1.6.39=h19919ed_0
-  - libprotobuf=3.21.12=h12be248_2
-  - libsodium=1.0.18=h62dcd97_0
-  - libsqlite=3.43.0=hcfcfb64_0
-  - libssh2=1.11.0=h7dfc565_0
-  - libthrift=0.18.1=h06f6336_2
-  - libtiff=4.4.0=hc4f729c_5
-  - libutf8proc=2.8.0=h82a8f57_0
-  - libuv=1.44.2=hcfcfb64_1
-  - libvorbis=1.3.7=h0e60522_0
-  - libwebp-base=1.3.2=hcfcfb64_0
-  - libxcb=1.13=hcd874cb_1004
-  - libxml2=2.11.5=hc3477c8_1
-  - libzlib=1.2.13=hcfcfb64_5
-  - lz4-c=1.9.4=hcfcfb64_0
-  - m2w64-gcc-libgfortran=5.3.0=6
-  - m2w64-gcc-libs=5.3.0=7
-  - m2w64-gcc-libs-core=5.3.0=7
-  - m2w64-gmp=6.1.0=2
-  - m2w64-libwinpthread-git=5.0.0.4634.697f757=2
-  - markupsafe=2.1.3=py39ha55989b_1
-  - matplotlib-base=3.8.0=py39hf19769e_1
-  - matplotlib-inline=0.1.6=py39haa95532_0
-  - mkl=2022.1.0=h6a75c08_874
-  - mkl-devel=2022.1.0=h57928b3_875
-  - mkl-include=2022.1.0=h6a75c08_874
-  - mpmath=1.3.0=pyhd8ed1ab_0
-  - msys2-conda-epoch=20160418=1
-  - multidict=6.0.4=py39ha55989b_0
-  - multiprocess=0.70.15=py39ha55989b_1
-  - munkres=1.1.4=pyh9f0ad1d_0
-  - nest-asyncio=1.5.6=py39haa95532_0
-  - networkx=3.1=pyhd8ed1ab_0
-  - numpy=1.26.0=py39hddb5d58_0
-  - opencv=4.5.3=py39hcbf5309_8
-  - openjpeg=2.5.0=hc9384bd_1
-  - openssl=3.1.3=hcfcfb64_0
-  - orc=1.9.0=hada7b9e_1
-  - packaging=23.1=pyhd8ed1ab_0
-  - pandas=2.1.1=py39h32e6231_0
-  - parso=0.8.3=pyhd3eb1b0_0
-  - pcre2=10.40=h17e33f8_0
-  - pickleshare=0.7.5=pyhd3eb1b0_1003
-  - pillow=9.2.0=py39h595c93f_3
-  - pip=23.2.1=pyhd8ed1ab_0
-  - platformdirs=3.10.0=pyhd8ed1ab_0
-  - prompt-toolkit=3.0.36=py39haa95532_0
-  - psutil=5.9.0=py39h2bbff1b_0
-  - pthread-stubs=0.4=hcd874cb_1001
-  - pthreads-win32=2.9.1=hfa6e2cd_3
-  - pure_eval=0.2.2=pyhd3eb1b0_0
-  - py-opencv=4.5.3=py39h00e5391_8
-  - pyarrow=12.0.1=py39hca4e8af_5_cpu
-  - pycocotools=2.0.6=py39hc266a54_1
-  - pygments=2.15.1=py39haa95532_1
-  - pyparsing=3.1.1=pyhd8ed1ab_0
-  - pysocks=1.7.1=pyh0701188_6
-  - python=3.9.18=h4de0772_0_cpython
-  - python-dateutil=2.8.2=pyhd8ed1ab_0
-  - python-tzdata=2023.3=pyhd8ed1ab_0
-  - python-xxhash=3.3.0=py39ha55989b_1
-  - python_abi=3.9=4_cp39
-  - pytorch=2.0.1=py3.9_cuda11.8_cudnn8_0
-  - pytorch-cuda=11.8=h24eeafa_5
-  - pytorch-mutex=1.0=cuda
-  - pytz=2023.3.post1=pyhd8ed1ab_0
-  - pywin32=305=py39h2bbff1b_0
-  - pyyaml=6.0.1=py39ha55989b_1
-  - pyzmq=25.1.0=py39hd77b12b_0
-  - qt-main=5.15.8=h720456b_6
-  - re2=2023.03.02=hd4eee63_0
-  - regex=2023.8.8=py39ha55989b_1
-  - requests=2.31.0=pyhd8ed1ab_0
-  - sacremoses=0.0.53=pyhd8ed1ab_0
-  - safetensors=0.3.3=py39hf21820d_1
-  - setuptools=68.2.2=pyhd8ed1ab_0
-  - six=1.16.0=pyh6c4a22f_0
-  - snappy=1.1.10=hfb803bf_0
-  - stack_data=0.2.0=pyhd3eb1b0_0
-  - sympy=1.12=pyh04b8f61_3
-  - tbb=2021.10.0=h91493d7_1
-  - timm=0.9.7=pyhd8ed1ab_0
-  - tk=8.6.13=hcfcfb64_0
-  - tokenizers=0.13.3=py39hca44cb7_0
-  - tomli=2.0.1=pyhd8ed1ab_0
-  - tornado=6.3.2=py39h2bbff1b_0
-  - tqdm=4.66.1=pyhd8ed1ab_0
-  - traitlets=5.7.1=py39haa95532_0
-  - transformers=4.33.2=pyhd8ed1ab_0
-  - typing-extensions=4.8.0=hd8ed1ab_0
-  - typing_extensions=4.8.0=pyha770c72_0
-  - tzdata=2023c=h71feb2d_0
-  - ucrt=10.0.22621.0=h57928b3_0
-  - unicodedata2=15.0.0=py39ha55989b_1
-  - urllib3=2.0.5=pyhd8ed1ab_0
-  - vc=14.3=h64f974e_17
-  - vc14_runtime=14.36.32532=hdcecf7f_17
-  - vs2015_runtime=14.36.32532=h05e6639_17
-  - wcwidth=0.2.5=pyhd3eb1b0_0
-  - wheel=0.41.2=pyhd8ed1ab_0
-  - win_inet_pton=1.1.0=pyhd8ed1ab_6
-  - xorg-libxau=1.0.11=hcd874cb_0
-  - xorg-libxdmcp=1.1.3=hcd874cb_0
-  - xxhash=0.8.2=hcfcfb64_0
-  - xz=5.2.6=h8d14728_0
-  - yaml=0.2.5=h8ffe710_2
-  - yapf=0.40.1=pyhd8ed1ab_0
-  - yarl=1.9.2=py39ha55989b_0
-  - zeromq=4.3.4=hd77b12b_0
-  - zipp=3.17.0=pyhd8ed1ab_0
-  - zlib=1.2.13=hcfcfb64_5
-  - zstd=1.5.5=h12be248_0
-  - pip:
-      - opencv-python==4.8.0.76
-      - supervision==0.6.0
-      - torchaudio==2.0.2
-      - torchvision==0.15.2
-prefix: C:\Users\Makoto\miniconda3\envs\dino

grounding_dino/groundingdino/config/GroundingDINO_SwinB_cfg.py

@@ -1,43 +0,0 @@
-batch_size = 1
-modelname = "groundingdino"
-backbone = "swin_B_384_22k"
-position_embedding = "sine"
-pe_temperatureH = 20
-pe_temperatureW = 20
-return_interm_indices = [1, 2, 3]
-backbone_freeze_keywords = None
-enc_layers = 6
-dec_layers = 6
-pre_norm = False
-dim_feedforward = 2048
-hidden_dim = 256
-dropout = 0.0
-nheads = 8
-num_queries = 900
-query_dim = 4
-num_patterns = 0
-num_feature_levels = 4
-enc_n_points = 4
-dec_n_points = 4
-two_stage_type = "standard"
-two_stage_bbox_embed_share = False
-two_stage_class_embed_share = False
-transformer_activation = "relu"
-dec_pred_bbox_embed_share = True
-dn_box_noise_scale = 1.0
-dn_label_noise_ratio = 0.5
-dn_label_coef = 1.0
-dn_bbox_coef = 1.0
-embed_init_tgt = True
-dn_labelbook_size = 2000
-max_text_len = 256
-text_encoder_type = "bert-base-uncased"
-use_text_enhancer = True
-use_fusion_layer = True
-use_checkpoint = True
-use_transformer_ckpt = True
-use_text_cross_attention = True
-text_dropout = 0.0
-fusion_dropout = 0.0
-fusion_droppath = 0.1
-sub_sentence_present = True

grounding_dino/groundingdino/config/GroundingDINO_SwinT_OGC.py

@@ -1,43 +0,0 @@
-batch_size = 1
-modelname = "groundingdino"
-backbone = "swin_T_224_1k"
-position_embedding = "sine"
-pe_temperatureH = 20
-pe_temperatureW = 20
-return_interm_indices = [1, 2, 3]
-backbone_freeze_keywords = None
-enc_layers = 6
-dec_layers = 6
-pre_norm = False
-dim_feedforward = 2048
-hidden_dim = 256
-dropout = 0.0
-nheads = 8
-num_queries = 900
-query_dim = 4
-num_patterns = 0
-num_feature_levels = 4
-enc_n_points = 4
-dec_n_points = 4
-two_stage_type = "standard"
-two_stage_bbox_embed_share = False
-two_stage_class_embed_share = False
-transformer_activation = "relu"
-dec_pred_bbox_embed_share = True
-dn_box_noise_scale = 1.0
-dn_label_noise_ratio = 0.5
-dn_label_coef = 1.0
-dn_bbox_coef = 1.0
-embed_init_tgt = True
-dn_labelbook_size = 2000
-max_text_len = 256
-text_encoder_type = "bert-base-uncased"
-use_text_enhancer = True
-use_fusion_layer = True
-use_checkpoint = True
-use_transformer_ckpt = True
-use_text_cross_attention = True
-text_dropout = 0.0
-fusion_dropout = 0.0
-fusion_droppath = 0.1
-sub_sentence_present = True

grounding_dino/groundingdino/datasets/cocogrounding_eval.py

@@ -1,269 +0,0 @@
-# ------------------------------------------------------------------------
-# Grounding DINO. Midified by Shilong Liu.
-# url: https://github.com/IDEA-Research/GroundingDINO
-# Copyright (c) 2023 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# Copyright (c) Aishwarya Kamath & Nicolas Carion. Licensed under the Apache License 2.0. All Rights Reserved
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-"""
-COCO evaluator that works in distributed mode.
-
-Mostly copy-paste from https://github.com/pytorch/vision/blob/edfd5a7/references/detection/coco_eval.py
-The difference is that there is less copy-pasting from pycocotools
-in the end of the file, as python3 can suppress prints with contextlib
-"""
-import contextlib
-import copy
-import os
-
-import numpy as np
-import pycocotools.mask as mask_util
-import torch
-from pycocotools.coco import COCO
-from pycocotools.cocoeval import COCOeval
-
-from groundingdino.util.misc import all_gather
-
-
-class CocoGroundingEvaluator(object):
-    def __init__(self, coco_gt, iou_types, useCats=True):
-        assert isinstance(iou_types, (list, tuple))
-        coco_gt = copy.deepcopy(coco_gt)
-        self.coco_gt = coco_gt
-
-        self.iou_types = iou_types
-        self.coco_eval = {}
-        for iou_type in iou_types:
-            self.coco_eval[iou_type] = COCOeval(coco_gt, iouType=iou_type)
-            self.coco_eval[iou_type].useCats = useCats
-
-        self.img_ids = []
-        self.eval_imgs = {k: [] for k in iou_types}
-        self.useCats = useCats
-
-    def update(self, predictions):
-        img_ids = list(np.unique(list(predictions.keys())))
-        self.img_ids.extend(img_ids)
-
-        for iou_type in self.iou_types:
-            results = self.prepare(predictions, iou_type)
-
-            # suppress pycocotools prints
-            with open(os.devnull, "w") as devnull:
-                with contextlib.redirect_stdout(devnull):
-                    coco_dt = COCO.loadRes(self.coco_gt, results) if results else COCO()
-
-            coco_eval = self.coco_eval[iou_type]
-
-            coco_eval.cocoDt = coco_dt
-            coco_eval.params.imgIds = list(img_ids)
-            coco_eval.params.useCats = self.useCats
-            img_ids, eval_imgs = evaluate(coco_eval)
-
-            self.eval_imgs[iou_type].append(eval_imgs)
-
-    def synchronize_between_processes(self):
-        for iou_type in self.iou_types:
-            self.eval_imgs[iou_type] = np.concatenate(self.eval_imgs[iou_type], 2)
-            create_common_coco_eval(self.coco_eval[iou_type], self.img_ids, self.eval_imgs[iou_type])
-
-    def accumulate(self):
-        for coco_eval in self.coco_eval.values():
-            coco_eval.accumulate()
-
-    def summarize(self):
-        for iou_type, coco_eval in self.coco_eval.items():
-            print("IoU metric: {}".format(iou_type))
-            coco_eval.summarize()
-
-    def prepare(self, predictions, iou_type):
-        if iou_type == "bbox":
-            return self.prepare_for_coco_detection(predictions)
-        elif iou_type == "segm":
-            return self.prepare_for_coco_segmentation(predictions)
-        elif iou_type == "keypoints":
-            return self.prepare_for_coco_keypoint(predictions)
-        else:
-            raise ValueError("Unknown iou type {}".format(iou_type))
-
-    def prepare_for_coco_detection(self, predictions):
-        coco_results = []
-        for original_id, prediction in predictions.items():
-            if len(prediction) == 0:
-                continue
-
-            boxes = prediction["boxes"]
-            boxes = convert_to_xywh(boxes).tolist()
-            scores = prediction["scores"].tolist()
-            labels = prediction["labels"].tolist()
-
-            coco_results.extend(
-                [
-                    {
-                        "image_id": original_id,
-                        "category_id": labels[k],
-                        "bbox": box,
-                        "score": scores[k],
-                    }
-                    for k, box in enumerate(boxes)
-                ]
-            )
-        return coco_results
-
-    def prepare_for_coco_segmentation(self, predictions):
-        coco_results = []
-        for original_id, prediction in predictions.items():
-            if len(prediction) == 0:
-                continue
-
-            scores = prediction["scores"]
-            labels = prediction["labels"]
-            masks = prediction["masks"]
-
-            masks = masks > 0.5
-
-            scores = prediction["scores"].tolist()
-            labels = prediction["labels"].tolist()
-
-            rles = [
-                mask_util.encode(np.array(mask[0, :, :, np.newaxis], dtype=np.uint8, order="F"))[0] 
-                for mask in masks
-            ]
-            for rle in rles:
-                rle["counts"] = rle["counts"].decode("utf-8")
-
-            coco_results.extend(
-                [
-                    {
-                        "image_id": original_id,
-                        "category_id": labels[k],
-                        "segmentation": rle,
-                        "score": scores[k],
-                    }
-                    for k, rle in enumerate(rles)
-                ]
-            )
-        return coco_results
-
-    def prepare_for_coco_keypoint(self, predictions):
-        coco_results = []
-        for original_id, prediction in predictions.items():
-            if len(prediction) == 0:
-                continue
-
-            boxes = prediction["boxes"]
-            boxes = convert_to_xywh(boxes).tolist()
-            scores = prediction["scores"].tolist()
-            labels = prediction["labels"].tolist()
-            keypoints = prediction["keypoints"]
-            keypoints = keypoints.flatten(start_dim=1).tolist()
-
-            coco_results.extend(
-                [
-                    {
-                        "image_id": original_id,
-                        "category_id": labels[k],
-                        "keypoints": keypoint,
-                        "score": scores[k],
-                    }
-                    for k, keypoint in enumerate(keypoints)
-                ]
-            )
-        return coco_results
-
-
-def convert_to_xywh(boxes):
-    xmin, ymin, xmax, ymax = boxes.unbind(1)
-    return torch.stack((xmin, ymin, xmax - xmin, ymax - ymin), dim=1)
-
-
-def merge(img_ids, eval_imgs):
-    all_img_ids = all_gather(img_ids)
-    all_eval_imgs = all_gather(eval_imgs)
-
-    merged_img_ids = []
-    for p in all_img_ids:
-        merged_img_ids.extend(p)
-
-    merged_eval_imgs = []
-    for p in all_eval_imgs:
-        merged_eval_imgs.append(p)
-
-    merged_img_ids = np.array(merged_img_ids)
-    merged_eval_imgs = np.concatenate(merged_eval_imgs, 2)
-
-    # keep only unique (and in sorted order) images
-    merged_img_ids, idx = np.unique(merged_img_ids, return_index=True)
-    merged_eval_imgs = merged_eval_imgs[..., idx]
-
-    return merged_img_ids, merged_eval_imgs
-
-
-def create_common_coco_eval(coco_eval, img_ids, eval_imgs):
-    img_ids, eval_imgs = merge(img_ids, eval_imgs)
-    img_ids = list(img_ids)
-    eval_imgs = list(eval_imgs.flatten())
-
-    coco_eval.evalImgs = eval_imgs
-    coco_eval.params.imgIds = img_ids
-    coco_eval._paramsEval = copy.deepcopy(coco_eval.params)
-
-
-#################################################################
-# From pycocotools, just removed the prints and fixed
-# a Python3 bug about unicode not defined
-#################################################################
-
-
-def evaluate(self):
-    """
-    Run per image evaluation on given images and store results (a list of dict) in self.evalImgs
-    :return: None
-    """
-    # tic = time.time()
-    # print('Running per image evaluation...')
-    p = self.params
-    # add backward compatibility if useSegm is specified in params
-    if p.useSegm is not None:
-        p.iouType = "segm" if p.useSegm == 1 else "bbox"
-        print("useSegm (deprecated) is not None. Running {} evaluation".format(p.iouType))
-    # print('Evaluate annotation type *{}*'.format(p.iouType))
-    p.imgIds = list(np.unique(p.imgIds))
-    if p.useCats:
-        p.catIds = list(np.unique(p.catIds))
-    p.maxDets = sorted(p.maxDets)
-    self.params = p
-
-    self._prepare()
-    # loop through images, area range, max detection number
-    catIds = p.catIds if p.useCats else [-1]
-
-    if p.iouType == "segm" or p.iouType == "bbox":
-        computeIoU = self.computeIoU
-    elif p.iouType == "keypoints":
-        computeIoU = self.computeOks
-    self.ious = {
-        (imgId, catId): computeIoU(imgId, catId) 
-        for imgId in p.imgIds 
-        for catId in catIds}
-
-    evaluateImg = self.evaluateImg
-    maxDet = p.maxDets[-1]
-    evalImgs = [
-        evaluateImg(imgId, catId, areaRng, maxDet) 
-        for catId in catIds 
-        for areaRng in p.areaRng 
-        for imgId in p.imgIds
-    ]
-    # this is NOT in the pycocotools code, but could be done outside
-    evalImgs = np.asarray(evalImgs).reshape(len(catIds), len(p.areaRng), len(p.imgIds))
-    self._paramsEval = copy.deepcopy(self.params)
-    # toc = time.time()
-    # print('DONE (t={:0.2f}s).'.format(toc-tic))
-    return p.imgIds, evalImgs
-
-
-#################################################################
-# end of straight copy from pycocotools, just removing the prints
-#################################################################

grounding_dino/groundingdino/datasets/transforms.py

@@ -1,311 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-"""
-Transforms and data augmentation for both image + bbox.
-"""
-import os
-import random
-
-import PIL
-import torch
-import torchvision.transforms as T
-import torchvision.transforms.functional as F
-
-from grounding_dino.groundingdino.util.box_ops import box_xyxy_to_cxcywh
-from grounding_dino.groundingdino.util.misc import interpolate
-
-
-def crop(image, target, region):
-    cropped_image = F.crop(image, *region)
-
-    target = target.copy()
-    i, j, h, w = region
-
-    # should we do something wrt the original size?
-    target["size"] = torch.tensor([h, w])
-
-    fields = ["labels", "area", "iscrowd", "positive_map"]
-
-    if "boxes" in target:
-        boxes = target["boxes"]
-        max_size = torch.as_tensor([w, h], dtype=torch.float32)
-        cropped_boxes = boxes - torch.as_tensor([j, i, j, i])
-        cropped_boxes = torch.min(cropped_boxes.reshape(-1, 2, 2), max_size)
-        cropped_boxes = cropped_boxes.clamp(min=0)
-        area = (cropped_boxes[:, 1, :] - cropped_boxes[:, 0, :]).prod(dim=1)
-        target["boxes"] = cropped_boxes.reshape(-1, 4)
-        target["area"] = area
-        fields.append("boxes")
-
-    if "masks" in target:
-        # FIXME should we update the area here if there are no boxes?
-        target["masks"] = target["masks"][:, i : i + h, j : j + w]
-        fields.append("masks")
-
-    # remove elements for which the boxes or masks that have zero area
-    if "boxes" in target or "masks" in target:
-        # favor boxes selection when defining which elements to keep
-        # this is compatible with previous implementation
-        if "boxes" in target:
-            cropped_boxes = target["boxes"].reshape(-1, 2, 2)
-            keep = torch.all(cropped_boxes[:, 1, :] > cropped_boxes[:, 0, :], dim=1)
-        else:
-            keep = target["masks"].flatten(1).any(1)
-
-        for field in fields:
-            if field in target:
-                target[field] = target[field][keep]
-
-    if os.environ.get("IPDB_SHILONG_DEBUG", None) == "INFO":
-        # for debug and visualization only.
-        if "strings_positive" in target:
-            target["strings_positive"] = [
-                _i for _i, _j in zip(target["strings_positive"], keep) if _j
-            ]
-
-    return cropped_image, target
-
-
-def hflip(image, target):
-    flipped_image = F.hflip(image)
-
-    w, h = image.size
-
-    target = target.copy()
-    if "boxes" in target:
-        boxes = target["boxes"]
-        boxes = boxes[:, [2, 1, 0, 3]] * torch.as_tensor([-1, 1, -1, 1]) + torch.as_tensor(
-            [w, 0, w, 0]
-        )
-        target["boxes"] = boxes
-
-    if "masks" in target:
-        target["masks"] = target["masks"].flip(-1)
-
-    return flipped_image, target
-
-
-def resize(image, target, size, max_size=None):
-    # size can be min_size (scalar) or (w, h) tuple
-
-    def get_size_with_aspect_ratio(image_size, size, max_size=None):
-        w, h = image_size
-        if max_size is not None:
-            min_original_size = float(min((w, h)))
-            max_original_size = float(max((w, h)))
-            if max_original_size / min_original_size * size > max_size:
-                size = int(round(max_size * min_original_size / max_original_size))
-
-        if (w <= h and w == size) or (h <= w and h == size):
-            return (h, w)
-
-        if w < h:
-            ow = size
-            oh = int(size * h / w)
-        else:
-            oh = size
-            ow = int(size * w / h)
-
-        return (oh, ow)
-
-    def get_size(image_size, size, max_size=None):
-        if isinstance(size, (list, tuple)):
-            return size[::-1]
-        else:
-            return get_size_with_aspect_ratio(image_size, size, max_size)
-
-    size = get_size(image.size, size, max_size)
-    rescaled_image = F.resize(image, size)
-
-    if target is None:
-        return rescaled_image, None
-
-    ratios = tuple(float(s) / float(s_orig) for s, s_orig in zip(rescaled_image.size, image.size))
-    ratio_width, ratio_height = ratios
-
-    target = target.copy()
-    if "boxes" in target:
-        boxes = target["boxes"]
-        scaled_boxes = boxes * torch.as_tensor(
-            [ratio_width, ratio_height, ratio_width, ratio_height]
-        )
-        target["boxes"] = scaled_boxes
-
-    if "area" in target:
-        area = target["area"]
-        scaled_area = area * (ratio_width * ratio_height)
-        target["area"] = scaled_area
-
-    h, w = size
-    target["size"] = torch.tensor([h, w])
-
-    if "masks" in target:
-        target["masks"] = (
-            interpolate(target["masks"][:, None].float(), size, mode="nearest")[:, 0] > 0.5
-        )
-
-    return rescaled_image, target
-
-
-def pad(image, target, padding):
-    # assumes that we only pad on the bottom right corners
-    padded_image = F.pad(image, (0, 0, padding[0], padding[1]))
-    if target is None:
-        return padded_image, None
-    target = target.copy()
-    # should we do something wrt the original size?
-    target["size"] = torch.tensor(padded_image.size[::-1])
-    if "masks" in target:
-        target["masks"] = torch.nn.functional.pad(target["masks"], (0, padding[0], 0, padding[1]))
-    return padded_image, target
-
-
-class ResizeDebug(object):
-    def __init__(self, size):
-        self.size = size
-
-    def __call__(self, img, target):
-        return resize(img, target, self.size)
-
-
-class RandomCrop(object):
-    def __init__(self, size):
-        self.size = size
-
-    def __call__(self, img, target):
-        region = T.RandomCrop.get_params(img, self.size)
-        return crop(img, target, region)
-
-
-class RandomSizeCrop(object):
-    def __init__(self, min_size: int, max_size: int, respect_boxes: bool = False):
-        # respect_boxes:    True to keep all boxes
-        #                   False to tolerence box filter
-        self.min_size = min_size
-        self.max_size = max_size
-        self.respect_boxes = respect_boxes
-
-    def __call__(self, img: PIL.Image.Image, target: dict):
-        init_boxes = len(target["boxes"])
-        max_patience = 10
-        for i in range(max_patience):
-            w = random.randint(self.min_size, min(img.width, self.max_size))
-            h = random.randint(self.min_size, min(img.height, self.max_size))
-            region = T.RandomCrop.get_params(img, [h, w])
-            result_img, result_target = crop(img, target, region)
-            if (
-                not self.respect_boxes
-                or len(result_target["boxes"]) == init_boxes
-                or i == max_patience - 1
-            ):
-                return result_img, result_target
-        return result_img, result_target
-
-
-class CenterCrop(object):
-    def __init__(self, size):
-        self.size = size
-
-    def __call__(self, img, target):
-        image_width, image_height = img.size
-        crop_height, crop_width = self.size
-        crop_top = int(round((image_height - crop_height) / 2.0))
-        crop_left = int(round((image_width - crop_width) / 2.0))
-        return crop(img, target, (crop_top, crop_left, crop_height, crop_width))
-
-
-class RandomHorizontalFlip(object):
-    def __init__(self, p=0.5):
-        self.p = p
-
-    def __call__(self, img, target):
-        if random.random() < self.p:
-            return hflip(img, target)
-        return img, target
-
-
-class RandomResize(object):
-    def __init__(self, sizes, max_size=None):
-        assert isinstance(sizes, (list, tuple))
-        self.sizes = sizes
-        self.max_size = max_size
-
-    def __call__(self, img, target=None):
-        size = random.choice(self.sizes)
-        return resize(img, target, size, self.max_size)
-
-
-class RandomPad(object):
-    def __init__(self, max_pad):
-        self.max_pad = max_pad
-
-    def __call__(self, img, target):
-        pad_x = random.randint(0, self.max_pad)
-        pad_y = random.randint(0, self.max_pad)
-        return pad(img, target, (pad_x, pad_y))
-
-
-class RandomSelect(object):
-    """
-    Randomly selects between transforms1 and transforms2,
-    with probability p for transforms1 and (1 - p) for transforms2
-    """
-
-    def __init__(self, transforms1, transforms2, p=0.5):
-        self.transforms1 = transforms1
-        self.transforms2 = transforms2
-        self.p = p
-
-    def __call__(self, img, target):
-        if random.random() < self.p:
-            return self.transforms1(img, target)
-        return self.transforms2(img, target)
-
-
-class ToTensor(object):
-    def __call__(self, img, target):
-        return F.to_tensor(img), target
-
-
-class RandomErasing(object):
-    def __init__(self, *args, **kwargs):
-        self.eraser = T.RandomErasing(*args, **kwargs)
-
-    def __call__(self, img, target):
-        return self.eraser(img), target
-
-
-class Normalize(object):
-    def __init__(self, mean, std):
-        self.mean = mean
-        self.std = std
-
-    def __call__(self, image, target=None):
-        image = F.normalize(image, mean=self.mean, std=self.std)
-        if target is None:
-            return image, None
-        target = target.copy()
-        h, w = image.shape[-2:]
-        if "boxes" in target:
-            boxes = target["boxes"]
-            boxes = box_xyxy_to_cxcywh(boxes)
-            boxes = boxes / torch.tensor([w, h, w, h], dtype=torch.float32)
-            target["boxes"] = boxes
-        return image, target
-
-
-class Compose(object):
-    def __init__(self, transforms):
-        self.transforms = transforms
-
-    def __call__(self, image, target):
-        for t in self.transforms:
-            image, target = t(image, target)
-        return image, target
-
-    def __repr__(self):
-        format_string = self.__class__.__name__ + "("
-        for t in self.transforms:
-            format_string += "\n"
-            format_string += "    {0}".format(t)
-        format_string += "\n)"
-        return format_string

grounding_dino/groundingdino/models/GroundingDINO/__init__.py

@@ -1,15 +0,0 @@
-# ------------------------------------------------------------------------
-# Grounding DINO
-# url: https://github.com/IDEA-Research/GroundingDINO
-# Copyright (c) 2023 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# Conditional DETR
-# Copyright (c) 2021 Microsoft. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# Copied from DETR (https://github.com/facebookresearch/detr)
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-# ------------------------------------------------------------------------
-
-from .groundingdino import build_groundingdino

grounding_dino/groundingdino/models/GroundingDINO/backbone/__init__.py

@@ -1 +0,0 @@
-from .backbone import build_backbone

grounding_dino/groundingdino/models/GroundingDINO/backbone/backbone.py

@@ -1,221 +0,0 @@
-# ------------------------------------------------------------------------
-# Grounding DINO
-# url: https://github.com/IDEA-Research/GroundingDINO
-# Copyright (c) 2023 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# Conditional DETR
-# Copyright (c) 2021 Microsoft. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# Copied from DETR (https://github.com/facebookresearch/detr)
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-# ------------------------------------------------------------------------
-
-"""
-Backbone modules.
-"""
-
-from typing import Dict, List
-
-import torch
-import torch.nn.functional as F
-import torchvision
-from torch import nn
-from torchvision.models._utils import IntermediateLayerGetter
-
-from grounding_dino.groundingdino.util.misc import NestedTensor, clean_state_dict, is_main_process
-
-from .position_encoding import build_position_encoding
-from .swin_transformer import build_swin_transformer
-
-
-class FrozenBatchNorm2d(torch.nn.Module):
-    """
-    BatchNorm2d where the batch statistics and the affine parameters are fixed.
-
-    Copy-paste from torchvision.misc.ops with added eps before rqsrt,
-    without which any other models than torchvision.models.resnet[18,34,50,101]
-    produce nans.
-    """
-
-    def __init__(self, n):
-        super(FrozenBatchNorm2d, self).__init__()
-        self.register_buffer("weight", torch.ones(n))
-        self.register_buffer("bias", torch.zeros(n))
-        self.register_buffer("running_mean", torch.zeros(n))
-        self.register_buffer("running_var", torch.ones(n))
-
-    def _load_from_state_dict(
-        self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs
-    ):
-        num_batches_tracked_key = prefix + "num_batches_tracked"
-        if num_batches_tracked_key in state_dict:
-            del state_dict[num_batches_tracked_key]
-
-        super(FrozenBatchNorm2d, self)._load_from_state_dict(
-            state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs
-        )
-
-    def forward(self, x):
-        # move reshapes to the beginning
-        # to make it fuser-friendly
-        w = self.weight.reshape(1, -1, 1, 1)
-        b = self.bias.reshape(1, -1, 1, 1)
-        rv = self.running_var.reshape(1, -1, 1, 1)
-        rm = self.running_mean.reshape(1, -1, 1, 1)
-        eps = 1e-5
-        scale = w * (rv + eps).rsqrt()
-        bias = b - rm * scale
-        return x * scale + bias
-
-
-class BackboneBase(nn.Module):
-    def __init__(
-        self,
-        backbone: nn.Module,
-        train_backbone: bool,
-        num_channels: int,
-        return_interm_indices: list,
-    ):
-        super().__init__()
-        for name, parameter in backbone.named_parameters():
-            if (
-                not train_backbone
-                or "layer2" not in name
-                and "layer3" not in name
-                and "layer4" not in name
-            ):
-                parameter.requires_grad_(False)
-
-        return_layers = {}
-        for idx, layer_index in enumerate(return_interm_indices):
-            return_layers.update(
-                {"layer{}".format(5 - len(return_interm_indices) + idx): "{}".format(layer_index)}
-            )
-
-        # if len:
-        #     if use_stage1_feature:
-        #         return_layers = {"layer1": "0", "layer2": "1", "layer3": "2", "layer4": "3"}
-        #     else:
-        #         return_layers = {"layer2": "0", "layer3": "1", "layer4": "2"}
-        # else:
-        #     return_layers = {'layer4': "0"}
-        self.body = IntermediateLayerGetter(backbone, return_layers=return_layers)
-        self.num_channels = num_channels
-
-    def forward(self, tensor_list: NestedTensor):
-        xs = self.body(tensor_list.tensors)
-        out: Dict[str, NestedTensor] = {}
-        for name, x in xs.items():
-            m = tensor_list.mask
-            assert m is not None
-            mask = F.interpolate(m[None].float(), size=x.shape[-2:]).to(torch.bool)[0]
-            out[name] = NestedTensor(x, mask)
-        # import ipdb; ipdb.set_trace()
-        return out
-
-
-class Backbone(BackboneBase):
-    """ResNet backbone with frozen BatchNorm."""
-
-    def __init__(
-        self,
-        name: str,
-        train_backbone: bool,
-        dilation: bool,
-        return_interm_indices: list,
-        batch_norm=FrozenBatchNorm2d,
-    ):
-        if name in ["resnet18", "resnet34", "resnet50", "resnet101"]:
-            backbone = getattr(torchvision.models, name)(
-                replace_stride_with_dilation=[False, False, dilation],
-                pretrained=is_main_process(),
-                norm_layer=batch_norm,
-            )
-        else:
-            raise NotImplementedError("Why you can get here with name {}".format(name))
-        # num_channels = 512 if name in ('resnet18', 'resnet34') else 2048
-        assert name not in ("resnet18", "resnet34"), "Only resnet50 and resnet101 are available."
-        assert return_interm_indices in [[0, 1, 2, 3], [1, 2, 3], [3]]
-        num_channels_all = [256, 512, 1024, 2048]
-        num_channels = num_channels_all[4 - len(return_interm_indices) :]
-        super().__init__(backbone, train_backbone, num_channels, return_interm_indices)
-
-
-class Joiner(nn.Sequential):
-    def __init__(self, backbone, position_embedding):
-        super().__init__(backbone, position_embedding)
-
-    def forward(self, tensor_list: NestedTensor):
-        xs = self[0](tensor_list)
-        out: List[NestedTensor] = []
-        pos = []
-        for name, x in xs.items():
-            out.append(x)
-            # position encoding
-            pos.append(self[1](x).to(x.tensors.dtype))
-
-        return out, pos
-
-
-def build_backbone(args):
-    """
-    Useful args:
-        - backbone: backbone name
-        - lr_backbone:
-        - dilation
-        - return_interm_indices: available: [0,1,2,3], [1,2,3], [3]
-        - backbone_freeze_keywords:
-        - use_checkpoint: for swin only for now
-
-    """
-    position_embedding = build_position_encoding(args)
-    train_backbone = True
-    if not train_backbone:
-        raise ValueError("Please set lr_backbone > 0")
-    return_interm_indices = args.return_interm_indices
-    assert return_interm_indices in [[0, 1, 2, 3], [1, 2, 3], [3]]
-    args.backbone_freeze_keywords
-    use_checkpoint = getattr(args, "use_checkpoint", False)
-
-    if args.backbone in ["resnet50", "resnet101"]:
-        backbone = Backbone(
-            args.backbone,
-            train_backbone,
-            args.dilation,
-            return_interm_indices,
-            batch_norm=FrozenBatchNorm2d,
-        )
-        bb_num_channels = backbone.num_channels
-    elif args.backbone in [
-        "swin_T_224_1k",
-        "swin_B_224_22k",
-        "swin_B_384_22k",
-        "swin_L_224_22k",
-        "swin_L_384_22k",
-    ]:
-        pretrain_img_size = int(args.backbone.split("_")[-2])
-        backbone = build_swin_transformer(
-            args.backbone,
-            pretrain_img_size=pretrain_img_size,
-            out_indices=tuple(return_interm_indices),
-            dilation=False,
-            use_checkpoint=use_checkpoint,
-        )
-
-        bb_num_channels = backbone.num_features[4 - len(return_interm_indices) :]
-    else:
-        raise NotImplementedError("Unknown backbone {}".format(args.backbone))
-
-    assert len(bb_num_channels) == len(
-        return_interm_indices
-    ), f"len(bb_num_channels) {len(bb_num_channels)} != len(return_interm_indices) {len(return_interm_indices)}"
-
-    model = Joiner(backbone, position_embedding)
-    model.num_channels = bb_num_channels
-    assert isinstance(
-        bb_num_channels, List
-    ), "bb_num_channels is expected to be a List but {}".format(type(bb_num_channels))
-    # import ipdb; ipdb.set_trace()
-    return model

grounding_dino/groundingdino/models/GroundingDINO/backbone/position_encoding.py

@@ -1,186 +0,0 @@
-# ------------------------------------------------------------------------
-# Grounding DINO
-# url: https://github.com/IDEA-Research/GroundingDINO
-# Copyright (c) 2023 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# DINO
-# Copyright (c) 2022 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# Conditional DETR
-# Copyright (c) 2021 Microsoft. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# Copied from DETR (https://github.com/facebookresearch/detr)
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-# ------------------------------------------------------------------------
-
-"""
-Various positional encodings for the transformer.
-"""
-import math
-
-import torch
-from torch import nn
-
-from grounding_dino.groundingdino.util.misc import NestedTensor
-
-
-class PositionEmbeddingSine(nn.Module):
-    """
-    This is a more standard version of the position embedding, very similar to the one
-    used by the Attention is all you need paper, generalized to work on images.
-    """
-
-    def __init__(self, num_pos_feats=64, temperature=10000, normalize=False, scale=None):
-        super().__init__()
-        self.num_pos_feats = num_pos_feats
-        self.temperature = temperature
-        self.normalize = normalize
-        if scale is not None and normalize is False:
-            raise ValueError("normalize should be True if scale is passed")
-        if scale is None:
-            scale = 2 * math.pi
-        self.scale = scale
-
-    def forward(self, tensor_list: NestedTensor):
-        x = tensor_list.tensors
-        mask = tensor_list.mask
-        assert mask is not None
-        not_mask = ~mask
-        y_embed = not_mask.cumsum(1, dtype=torch.float32)
-        x_embed = not_mask.cumsum(2, dtype=torch.float32)
-        if self.normalize:
-            eps = 1e-6
-            # if os.environ.get("SHILONG_AMP", None) == '1':
-            #     eps = 1e-4
-            # else:
-            #     eps = 1e-6
-            y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale
-            x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale
-
-        dim_t = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device)
-        dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_pos_feats)
-
-        pos_x = x_embed[:, :, :, None] / dim_t
-        pos_y = y_embed[:, :, :, None] / dim_t
-        pos_x = torch.stack(
-            (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4
-        ).flatten(3)
-        pos_y = torch.stack(
-            (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4
-        ).flatten(3)
-        pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2)
-        return pos
-
-
-class PositionEmbeddingSineHW(nn.Module):
-    """
-    This is a more standard version of the position embedding, very similar to the one
-    used by the Attention is all you need paper, generalized to work on images.
-    """
-
-    def __init__(
-        self, num_pos_feats=64, temperatureH=10000, temperatureW=10000, normalize=False, scale=None
-    ):
-        super().__init__()
-        self.num_pos_feats = num_pos_feats
-        self.temperatureH = temperatureH
-        self.temperatureW = temperatureW
-        self.normalize = normalize
-        if scale is not None and normalize is False:
-            raise ValueError("normalize should be True if scale is passed")
-        if scale is None:
-            scale = 2 * math.pi
-        self.scale = scale
-
-    def forward(self, tensor_list: NestedTensor):
-        x = tensor_list.tensors
-        mask = tensor_list.mask
-        assert mask is not None
-        not_mask = ~mask
-        y_embed = not_mask.cumsum(1, dtype=torch.float32)
-        x_embed = not_mask.cumsum(2, dtype=torch.float32)
-
-        # import ipdb; ipdb.set_trace()
-
-        if self.normalize:
-            eps = 1e-6
-            y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale
-            x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale
-
-        dim_tx = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device)
-        dim_tx = self.temperatureW ** (2 * (torch.div(dim_tx, 2, rounding_mode='floor')) / self.num_pos_feats)
-        pos_x = x_embed[:, :, :, None] / dim_tx
-
-        dim_ty = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device)
-        dim_ty = self.temperatureH ** (2 * (torch.div(dim_ty, 2, rounding_mode='floor')) / self.num_pos_feats)
-        pos_y = y_embed[:, :, :, None] / dim_ty
-
-        pos_x = torch.stack(
-            (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4
-        ).flatten(3)
-        pos_y = torch.stack(
-            (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4
-        ).flatten(3)
-        pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2)
-
-        # import ipdb; ipdb.set_trace()
-
-        return pos
-
-
-class PositionEmbeddingLearned(nn.Module):
-    """
-    Absolute pos embedding, learned.
-    """
-
-    def __init__(self, num_pos_feats=256):
-        super().__init__()
-        self.row_embed = nn.Embedding(50, num_pos_feats)
-        self.col_embed = nn.Embedding(50, num_pos_feats)
-        self.reset_parameters()
-
-    def reset_parameters(self):
-        nn.init.uniform_(self.row_embed.weight)
-        nn.init.uniform_(self.col_embed.weight)
-
-    def forward(self, tensor_list: NestedTensor):
-        x = tensor_list.tensors
-        h, w = x.shape[-2:]
-        i = torch.arange(w, device=x.device)
-        j = torch.arange(h, device=x.device)
-        x_emb = self.col_embed(i)
-        y_emb = self.row_embed(j)
-        pos = (
-            torch.cat(
-                [
-                    x_emb.unsqueeze(0).repeat(h, 1, 1),
-                    y_emb.unsqueeze(1).repeat(1, w, 1),
-                ],
-                dim=-1,
-            )
-            .permute(2, 0, 1)
-            .unsqueeze(0)
-            .repeat(x.shape[0], 1, 1, 1)
-        )
-        return pos
-
-
-def build_position_encoding(args):
-    N_steps = args.hidden_dim // 2
-    if args.position_embedding in ("v2", "sine"):
-        # TODO find a better way of exposing other arguments
-        position_embedding = PositionEmbeddingSineHW(
-            N_steps,
-            temperatureH=args.pe_temperatureH,
-            temperatureW=args.pe_temperatureW,
-            normalize=True,
-        )
-    elif args.position_embedding in ("v3", "learned"):
-        position_embedding = PositionEmbeddingLearned(N_steps)
-    else:
-        raise ValueError(f"not supported {args.position_embedding}")
-
-    return position_embedding

grounding_dino/groundingdino/models/GroundingDINO/backbone/swin_transformer.py

@@ -1,802 +0,0 @@
-# ------------------------------------------------------------------------
-# Grounding DINO
-# url: https://github.com/IDEA-Research/GroundingDINO
-# Copyright (c) 2023 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# DINO
-# Copyright (c) 2022 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# --------------------------------------------------------
-# modified from https://github.com/SwinTransformer/Swin-Transformer-Object-Detection/blob/master/mmdet/models/backbones/swin_transformer.py
-# --------------------------------------------------------
-
-import numpy as np
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-import torch.utils.checkpoint as checkpoint
-from timm.models.layers import DropPath, to_2tuple, trunc_normal_
-
-from grounding_dino.groundingdino.util.misc import NestedTensor
-
-
-class Mlp(nn.Module):
-    """Multilayer perceptron."""
-
-    def __init__(
-        self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.0
-    ):
-        super().__init__()
-        out_features = out_features or in_features
-        hidden_features = hidden_features or in_features
-        self.fc1 = nn.Linear(in_features, hidden_features)
-        self.act = act_layer()
-        self.fc2 = nn.Linear(hidden_features, out_features)
-        self.drop = nn.Dropout(drop)
-
-    def forward(self, x):
-        x = self.fc1(x)
-        x = self.act(x)
-        x = self.drop(x)
-        x = self.fc2(x)
-        x = self.drop(x)
-        return x
-
-
-def window_partition(x, window_size):
-    """
-    Args:
-        x: (B, H, W, C)
-        window_size (int): window size
-    Returns:
-        windows: (num_windows*B, window_size, window_size, C)
-    """
-    B, H, W, C = x.shape
-    x = x.view(B, H // window_size, window_size, W // window_size, window_size, C)
-    windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)
-    return windows
-
-
-def window_reverse(windows, window_size, H, W):
-    """
-    Args:
-        windows: (num_windows*B, window_size, window_size, C)
-        window_size (int): Window size
-        H (int): Height of image
-        W (int): Width of image
-    Returns:
-        x: (B, H, W, C)
-    """
-    B = int(windows.shape[0] / (H * W / window_size / window_size))
-    x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1)
-    x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)
-    return x
-
-
-class WindowAttention(nn.Module):
-    """Window based multi-head self attention (W-MSA) module with relative position bias.
-    It supports both of shifted and non-shifted window.
-    Args:
-        dim (int): Number of input channels.
-        window_size (tuple[int]): The height and width of the window.
-        num_heads (int): Number of attention heads.
-        qkv_bias (bool, optional):  If True, add a learnable bias to query, key, value. Default: True
-        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set
-        attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0
-        proj_drop (float, optional): Dropout ratio of output. Default: 0.0
-    """
-
-    def __init__(
-        self,
-        dim,
-        window_size,
-        num_heads,
-        qkv_bias=True,
-        qk_scale=None,
-        attn_drop=0.0,
-        proj_drop=0.0,
-    ):
-
-        super().__init__()
-        self.dim = dim
-        self.window_size = window_size  # Wh, Ww
-        self.num_heads = num_heads
-        head_dim = dim // num_heads
-        self.scale = qk_scale or head_dim**-0.5
-
-        # define a parameter table of relative position bias
-        self.relative_position_bias_table = nn.Parameter(
-            torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads)
-        )  # 2*Wh-1 * 2*Ww-1, nH
-
-        # get pair-wise relative position index for each token inside the window
-        coords_h = torch.arange(self.window_size[0])
-        coords_w = torch.arange(self.window_size[1])
-        coords = torch.stack(torch.meshgrid([coords_h, coords_w]))  # 2, Wh, Ww
-        coords_flatten = torch.flatten(coords, 1)  # 2, Wh*Ww
-        relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :]  # 2, Wh*Ww, Wh*Ww
-        relative_coords = relative_coords.permute(1, 2, 0).contiguous()  # Wh*Ww, Wh*Ww, 2
-        relative_coords[:, :, 0] += self.window_size[0] - 1  # shift to start from 0
-        relative_coords[:, :, 1] += self.window_size[1] - 1
-        relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1
-        relative_position_index = relative_coords.sum(-1)  # Wh*Ww, Wh*Ww
-        self.register_buffer("relative_position_index", relative_position_index)
-
-        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
-        self.attn_drop = nn.Dropout(attn_drop)
-        self.proj = nn.Linear(dim, dim)
-        self.proj_drop = nn.Dropout(proj_drop)
-
-        trunc_normal_(self.relative_position_bias_table, std=0.02)
-        self.softmax = nn.Softmax(dim=-1)
-
-    def forward(self, x, mask=None):
-        """Forward function.
-        Args:
-            x: input features with shape of (num_windows*B, N, C)
-            mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None
-        """
-        B_, N, C = x.shape
-        qkv = (
-            self.qkv(x)
-            .reshape(B_, N, 3, self.num_heads, C // self.num_heads)
-            .permute(2, 0, 3, 1, 4)
-        )
-        q, k, v = qkv[0], qkv[1], qkv[2]  # make torchscript happy (cannot use tensor as tuple)
-
-        q = q * self.scale
-        attn = q @ k.transpose(-2, -1)
-
-        relative_position_bias = self.relative_position_bias_table[
-            self.relative_position_index.view(-1)
-        ].view(
-            self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1
-        )  # Wh*Ww,Wh*Ww,nH
-        relative_position_bias = relative_position_bias.permute(
-            2, 0, 1
-        ).contiguous()  # nH, Wh*Ww, Wh*Ww
-        attn = attn + relative_position_bias.unsqueeze(0)
-
-        if mask is not None:
-            nW = mask.shape[0]
-            attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0)
-            attn = attn.view(-1, self.num_heads, N, N)
-            attn = self.softmax(attn)
-        else:
-            attn = self.softmax(attn)
-
-        attn = self.attn_drop(attn)
-
-        x = (attn @ v).transpose(1, 2).reshape(B_, N, C)
-        x = self.proj(x)
-        x = self.proj_drop(x)
-        return x
-
-
-class SwinTransformerBlock(nn.Module):
-    """Swin Transformer Block.
-    Args:
-        dim (int): Number of input channels.
-        num_heads (int): Number of attention heads.
-        window_size (int): Window size.
-        shift_size (int): Shift size for SW-MSA.
-        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
-        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
-        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set.
-        drop (float, optional): Dropout rate. Default: 0.0
-        attn_drop (float, optional): Attention dropout rate. Default: 0.0
-        drop_path (float, optional): Stochastic depth rate. Default: 0.0
-        act_layer (nn.Module, optional): Activation layer. Default: nn.GELU
-        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
-    """
-
-    def __init__(
-        self,
-        dim,
-        num_heads,
-        window_size=7,
-        shift_size=0,
-        mlp_ratio=4.0,
-        qkv_bias=True,
-        qk_scale=None,
-        drop=0.0,
-        attn_drop=0.0,
-        drop_path=0.0,
-        act_layer=nn.GELU,
-        norm_layer=nn.LayerNorm,
-    ):
-        super().__init__()
-        self.dim = dim
-        self.num_heads = num_heads
-        self.window_size = window_size
-        self.shift_size = shift_size
-        self.mlp_ratio = mlp_ratio
-        assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size"
-
-        self.norm1 = norm_layer(dim)
-        self.attn = WindowAttention(
-            dim,
-            window_size=to_2tuple(self.window_size),
-            num_heads=num_heads,
-            qkv_bias=qkv_bias,
-            qk_scale=qk_scale,
-            attn_drop=attn_drop,
-            proj_drop=drop,
-        )
-
-        self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
-        self.norm2 = norm_layer(dim)
-        mlp_hidden_dim = int(dim * mlp_ratio)
-        self.mlp = Mlp(
-            in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop
-        )
-
-        self.H = None
-        self.W = None
-
-    def forward(self, x, mask_matrix):
-        """Forward function.
-        Args:
-            x: Input feature, tensor size (B, H*W, C).
-            H, W: Spatial resolution of the input feature.
-            mask_matrix: Attention mask for cyclic shift.
-        """
-        B, L, C = x.shape
-        H, W = self.H, self.W
-        assert L == H * W, "input feature has wrong size"
-
-        shortcut = x
-        x = self.norm1(x)
-        x = x.view(B, H, W, C)
-
-        # pad feature maps to multiples of window size
-        pad_l = pad_t = 0
-        pad_r = (self.window_size - W % self.window_size) % self.window_size
-        pad_b = (self.window_size - H % self.window_size) % self.window_size
-        x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b))
-        _, Hp, Wp, _ = x.shape
-
-        # cyclic shift
-        if self.shift_size > 0:
-            shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2))
-            attn_mask = mask_matrix
-        else:
-            shifted_x = x
-            attn_mask = None
-
-        # partition windows
-        x_windows = window_partition(
-            shifted_x, self.window_size
-        )  # nW*B, window_size, window_size, C
-        x_windows = x_windows.view(
-            -1, self.window_size * self.window_size, C
-        )  # nW*B, window_size*window_size, C
-
-        # W-MSA/SW-MSA
-        attn_windows = self.attn(x_windows, mask=attn_mask)  # nW*B, window_size*window_size, C
-
-        # merge windows
-        attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C)
-        shifted_x = window_reverse(attn_windows, self.window_size, Hp, Wp)  # B H' W' C
-
-        # reverse cyclic shift
-        if self.shift_size > 0:
-            x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2))
-        else:
-            x = shifted_x
-
-        if pad_r > 0 or pad_b > 0:
-            x = x[:, :H, :W, :].contiguous()
-
-        x = x.view(B, H * W, C)
-
-        # FFN
-        x = shortcut + self.drop_path(x)
-        x = x + self.drop_path(self.mlp(self.norm2(x)))
-
-        return x
-
-
-class PatchMerging(nn.Module):
-    """Patch Merging Layer
-    Args:
-        dim (int): Number of input channels.
-        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
-    """
-
-    def __init__(self, dim, norm_layer=nn.LayerNorm):
-        super().__init__()
-        self.dim = dim
-        self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False)
-        self.norm = norm_layer(4 * dim)
-
-    def forward(self, x, H, W):
-        """Forward function.
-        Args:
-            x: Input feature, tensor size (B, H*W, C).
-            H, W: Spatial resolution of the input feature.
-        """
-        B, L, C = x.shape
-        assert L == H * W, "input feature has wrong size"
-
-        x = x.view(B, H, W, C)
-
-        # padding
-        pad_input = (H % 2 == 1) or (W % 2 == 1)
-        if pad_input:
-            x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2))
-
-        x0 = x[:, 0::2, 0::2, :]  # B H/2 W/2 C
-        x1 = x[:, 1::2, 0::2, :]  # B H/2 W/2 C
-        x2 = x[:, 0::2, 1::2, :]  # B H/2 W/2 C
-        x3 = x[:, 1::2, 1::2, :]  # B H/2 W/2 C
-        x = torch.cat([x0, x1, x2, x3], -1)  # B H/2 W/2 4*C
-        x = x.view(B, -1, 4 * C)  # B H/2*W/2 4*C
-
-        x = self.norm(x)
-        x = self.reduction(x)
-
-        return x
-
-
-class BasicLayer(nn.Module):
-    """A basic Swin Transformer layer for one stage.
-    Args:
-        dim (int): Number of feature channels
-        depth (int): Depths of this stage.
-        num_heads (int): Number of attention head.
-        window_size (int): Local window size. Default: 7.
-        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4.
-        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
-        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set.
-        drop (float, optional): Dropout rate. Default: 0.0
-        attn_drop (float, optional): Attention dropout rate. Default: 0.0
-        drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0
-        norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm
-        downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None
-        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.
-    """
-
-    def __init__(
-        self,
-        dim,
-        depth,
-        num_heads,
-        window_size=7,
-        mlp_ratio=4.0,
-        qkv_bias=True,
-        qk_scale=None,
-        drop=0.0,
-        attn_drop=0.0,
-        drop_path=0.0,
-        norm_layer=nn.LayerNorm,
-        downsample=None,
-        use_checkpoint=False,
-    ):
-        super().__init__()
-        self.window_size = window_size
-        self.shift_size = window_size // 2
-        self.depth = depth
-        self.use_checkpoint = use_checkpoint
-
-        # build blocks
-        self.blocks = nn.ModuleList(
-            [
-                SwinTransformerBlock(
-                    dim=dim,
-                    num_heads=num_heads,
-                    window_size=window_size,
-                    shift_size=0 if (i % 2 == 0) else window_size // 2,
-                    mlp_ratio=mlp_ratio,
-                    qkv_bias=qkv_bias,
-                    qk_scale=qk_scale,
-                    drop=drop,
-                    attn_drop=attn_drop,
-                    drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path,
-                    norm_layer=norm_layer,
-                )
-                for i in range(depth)
-            ]
-        )
-
-        # patch merging layer
-        if downsample is not None:
-            self.downsample = downsample(dim=dim, norm_layer=norm_layer)
-        else:
-            self.downsample = None
-
-    def forward(self, x, H, W):
-        """Forward function.
-        Args:
-            x: Input feature, tensor size (B, H*W, C).
-            H, W: Spatial resolution of the input feature.
-        """
-
-        # calculate attention mask for SW-MSA
-        Hp = int(np.ceil(H / self.window_size)) * self.window_size
-        Wp = int(np.ceil(W / self.window_size)) * self.window_size
-        img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device)  # 1 Hp Wp 1
-        h_slices = (
-            slice(0, -self.window_size),
-            slice(-self.window_size, -self.shift_size),
-            slice(-self.shift_size, None),
-        )
-        w_slices = (
-            slice(0, -self.window_size),
-            slice(-self.window_size, -self.shift_size),
-            slice(-self.shift_size, None),
-        )
-        cnt = 0
-        for h in h_slices:
-            for w in w_slices:
-                img_mask[:, h, w, :] = cnt
-                cnt += 1
-
-        mask_windows = window_partition(
-            img_mask, self.window_size
-        )  # nW, window_size, window_size, 1
-        mask_windows = mask_windows.view(-1, self.window_size * self.window_size)
-        attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)
-        attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(
-            attn_mask == 0, float(0.0)
-        )
-
-        for blk in self.blocks:
-            blk.H, blk.W = H, W
-            if self.use_checkpoint:
-                x = checkpoint.checkpoint(blk, x, attn_mask)
-            else:
-                x = blk(x, attn_mask)
-        if self.downsample is not None:
-            x_down = self.downsample(x, H, W)
-            Wh, Ww = (H + 1) // 2, (W + 1) // 2
-            return x, H, W, x_down, Wh, Ww
-        else:
-            return x, H, W, x, H, W
-
-
-class PatchEmbed(nn.Module):
-    """Image to Patch Embedding
-    Args:
-        patch_size (int): Patch token size. Default: 4.
-        in_chans (int): Number of input image channels. Default: 3.
-        embed_dim (int): Number of linear projection output channels. Default: 96.
-        norm_layer (nn.Module, optional): Normalization layer. Default: None
-    """
-
-    def __init__(self, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None):
-        super().__init__()
-        patch_size = to_2tuple(patch_size)
-        self.patch_size = patch_size
-
-        self.in_chans = in_chans
-        self.embed_dim = embed_dim
-
-        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
-        if norm_layer is not None:
-            self.norm = norm_layer(embed_dim)
-        else:
-            self.norm = None
-
-    def forward(self, x):
-        """Forward function."""
-        # padding
-        _, _, H, W = x.size()
-        if W % self.patch_size[1] != 0:
-            x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1]))
-        if H % self.patch_size[0] != 0:
-            x = F.pad(x, (0, 0, 0, self.patch_size[0] - H % self.patch_size[0]))
-
-        x = self.proj(x)  # B C Wh Ww
-        if self.norm is not None:
-            Wh, Ww = x.size(2), x.size(3)
-            x = x.flatten(2).transpose(1, 2)
-            x = self.norm(x)
-            x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww)
-
-        return x
-
-
-class SwinTransformer(nn.Module):
-    """Swin Transformer backbone.
-        A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows`  -
-          https://arxiv.org/pdf/2103.14030
-    Args:
-        pretrain_img_size (int): Input image size for training the pretrained model,
-            used in absolute postion embedding. Default 224.
-        patch_size (int | tuple(int)): Patch size. Default: 4.
-        in_chans (int): Number of input image channels. Default: 3.
-        embed_dim (int): Number of linear projection output channels. Default: 96.
-        depths (tuple[int]): Depths of each Swin Transformer stage.
-        num_heads (tuple[int]): Number of attention head of each stage.
-        window_size (int): Window size. Default: 7.
-        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4.
-        qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True
-        qk_scale (float): Override default qk scale of head_dim ** -0.5 if set.
-        drop_rate (float): Dropout rate.
-        attn_drop_rate (float): Attention dropout rate. Default: 0.
-        drop_path_rate (float): Stochastic depth rate. Default: 0.2.
-        norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm.
-        ape (bool): If True, add absolute position embedding to the patch embedding. Default: False.
-        patch_norm (bool): If True, add normalization after patch embedding. Default: True.
-        out_indices (Sequence[int]): Output from which stages.
-        frozen_stages (int): Stages to be frozen (stop grad and set eval mode).
-            -1 means not freezing any parameters.
-        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.
-        dilation (bool): if True, the output size if 16x downsample, ow 32x downsample.
-    """
-
-    def __init__(
-        self,
-        pretrain_img_size=224,
-        patch_size=4,
-        in_chans=3,
-        embed_dim=96,
-        depths=[2, 2, 6, 2],
-        num_heads=[3, 6, 12, 24],
-        window_size=7,
-        mlp_ratio=4.0,
-        qkv_bias=True,
-        qk_scale=None,
-        drop_rate=0.0,
-        attn_drop_rate=0.0,
-        drop_path_rate=0.2,
-        norm_layer=nn.LayerNorm,
-        ape=False,
-        patch_norm=True,
-        out_indices=(0, 1, 2, 3),
-        frozen_stages=-1,
-        dilation=False,
-        use_checkpoint=False,
-    ):
-        super().__init__()
-
-        self.pretrain_img_size = pretrain_img_size
-        self.num_layers = len(depths)
-        self.embed_dim = embed_dim
-        self.ape = ape
-        self.patch_norm = patch_norm
-        self.out_indices = out_indices
-        self.frozen_stages = frozen_stages
-        self.dilation = dilation
-
-        # if use_checkpoint:
-        #     print("use_checkpoint!!!!!!!!!!!!!!!!!!!!!!!!")
-
-        # split image into non-overlapping patches
-        self.patch_embed = PatchEmbed(
-            patch_size=patch_size,
-            in_chans=in_chans,
-            embed_dim=embed_dim,
-            norm_layer=norm_layer if self.patch_norm else None,
-        )
-
-        # absolute position embedding
-        if self.ape:
-            pretrain_img_size = to_2tuple(pretrain_img_size)
-            patch_size = to_2tuple(patch_size)
-            patches_resolution = [
-                pretrain_img_size[0] // patch_size[0],
-                pretrain_img_size[1] // patch_size[1],
-            ]
-
-            self.absolute_pos_embed = nn.Parameter(
-                torch.zeros(1, embed_dim, patches_resolution[0], patches_resolution[1])
-            )
-            trunc_normal_(self.absolute_pos_embed, std=0.02)
-
-        self.pos_drop = nn.Dropout(p=drop_rate)
-
-        # stochastic depth
-        dpr = [
-            x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))
-        ]  # stochastic depth decay rule
-
-        # build layers
-        self.layers = nn.ModuleList()
-        # prepare downsample list
-        downsamplelist = [PatchMerging for i in range(self.num_layers)]
-        downsamplelist[-1] = None
-        num_features = [int(embed_dim * 2**i) for i in range(self.num_layers)]
-        if self.dilation:
-            downsamplelist[-2] = None
-            num_features[-1] = int(embed_dim * 2 ** (self.num_layers - 1)) // 2
-        for i_layer in range(self.num_layers):
-            layer = BasicLayer(
-                # dim=int(embed_dim * 2 ** i_layer),
-                dim=num_features[i_layer],
-                depth=depths[i_layer],
-                num_heads=num_heads[i_layer],
-                window_size=window_size,
-                mlp_ratio=mlp_ratio,
-                qkv_bias=qkv_bias,
-                qk_scale=qk_scale,
-                drop=drop_rate,
-                attn_drop=attn_drop_rate,
-                drop_path=dpr[sum(depths[:i_layer]) : sum(depths[: i_layer + 1])],
-                norm_layer=norm_layer,
-                # downsample=PatchMerging if (i_layer < self.num_layers - 1) else None,
-                downsample=downsamplelist[i_layer],
-                use_checkpoint=use_checkpoint,
-            )
-            self.layers.append(layer)
-
-        # num_features = [int(embed_dim * 2 ** i) for i in range(self.num_layers)]
-        self.num_features = num_features
-
-        # add a norm layer for each output
-        for i_layer in out_indices:
-            layer = norm_layer(num_features[i_layer])
-            layer_name = f"norm{i_layer}"
-            self.add_module(layer_name, layer)
-
-        self._freeze_stages()
-
-    def _freeze_stages(self):
-        if self.frozen_stages >= 0:
-            self.patch_embed.eval()
-            for param in self.patch_embed.parameters():
-                param.requires_grad = False
-
-        if self.frozen_stages >= 1 and self.ape:
-            self.absolute_pos_embed.requires_grad = False
-
-        if self.frozen_stages >= 2:
-            self.pos_drop.eval()
-            for i in range(0, self.frozen_stages - 1):
-                m = self.layers[i]
-                m.eval()
-                for param in m.parameters():
-                    param.requires_grad = False
-
-    # def init_weights(self, pretrained=None):
-    #     """Initialize the weights in backbone.
-    #     Args:
-    #         pretrained (str, optional): Path to pre-trained weights.
-    #             Defaults to None.
-    #     """
-
-    #     def _init_weights(m):
-    #         if isinstance(m, nn.Linear):
-    #             trunc_normal_(m.weight, std=.02)
-    #             if isinstance(m, nn.Linear) and m.bias is not None:
-    #                 nn.init.constant_(m.bias, 0)
-    #         elif isinstance(m, nn.LayerNorm):
-    #             nn.init.constant_(m.bias, 0)
-    #             nn.init.constant_(m.weight, 1.0)
-
-    #     if isinstance(pretrained, str):
-    #         self.apply(_init_weights)
-    #         logger = get_root_logger()
-    #         load_checkpoint(self, pretrained, strict=False, logger=logger)
-    #     elif pretrained is None:
-    #         self.apply(_init_weights)
-    #     else:
-    #         raise TypeError('pretrained must be a str or None')
-
-    def forward_raw(self, x):
-        """Forward function."""
-        x = self.patch_embed(x)
-
-        Wh, Ww = x.size(2), x.size(3)
-        if self.ape:
-            # interpolate the position embedding to the corresponding size
-            absolute_pos_embed = F.interpolate(
-                self.absolute_pos_embed, size=(Wh, Ww), mode="bicubic"
-            )
-            x = (x + absolute_pos_embed).flatten(2).transpose(1, 2)  # B Wh*Ww C
-        else:
-            x = x.flatten(2).transpose(1, 2)
-        x = self.pos_drop(x)
-
-        outs = []
-        for i in range(self.num_layers):
-            layer = self.layers[i]
-            x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww)
-            # import ipdb; ipdb.set_trace()
-
-            if i in self.out_indices:
-                norm_layer = getattr(self, f"norm{i}")
-                x_out = norm_layer(x_out)
-
-                out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous()
-                outs.append(out)
-        # in:
-        #   torch.Size([2, 3, 1024, 1024])
-        # outs:
-        #   [torch.Size([2, 192, 256, 256]), torch.Size([2, 384, 128, 128]), \
-        #       torch.Size([2, 768, 64, 64]), torch.Size([2, 1536, 32, 32])]
-        return tuple(outs)
-
-    def forward(self, tensor_list: NestedTensor):
-        x = tensor_list.tensors
-
-        """Forward function."""
-        x = self.patch_embed(x)
-
-        Wh, Ww = x.size(2), x.size(3)
-        if self.ape:
-            # interpolate the position embedding to the corresponding size
-            absolute_pos_embed = F.interpolate(
-                self.absolute_pos_embed, size=(Wh, Ww), mode="bicubic"
-            )
-            x = (x + absolute_pos_embed).flatten(2).transpose(1, 2)  # B Wh*Ww C
-        else:
-            x = x.flatten(2).transpose(1, 2)
-        x = self.pos_drop(x)
-
-        outs = []
-        for i in range(self.num_layers):
-            layer = self.layers[i]
-            x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww)
-
-            if i in self.out_indices:
-                norm_layer = getattr(self, f"norm{i}")
-                x_out = norm_layer(x_out)
-
-                out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous()
-                outs.append(out)
-        # in:
-        #   torch.Size([2, 3, 1024, 1024])
-        # out:
-        #   [torch.Size([2, 192, 256, 256]), torch.Size([2, 384, 128, 128]), \
-        #       torch.Size([2, 768, 64, 64]), torch.Size([2, 1536, 32, 32])]
-
-        # collect for nesttensors
-        outs_dict = {}
-        for idx, out_i in enumerate(outs):
-            m = tensor_list.mask
-            assert m is not None
-            mask = F.interpolate(m[None].float(), size=out_i.shape[-2:]).to(torch.bool)[0]
-            outs_dict[idx] = NestedTensor(out_i, mask)
-
-        return outs_dict
-
-    def train(self, mode=True):
-        """Convert the model into training mode while keep layers freezed."""
-        super(SwinTransformer, self).train(mode)
-        self._freeze_stages()
-
-
-def build_swin_transformer(modelname, pretrain_img_size, **kw):
-    assert modelname in [
-        "swin_T_224_1k",
-        "swin_B_224_22k",
-        "swin_B_384_22k",
-        "swin_L_224_22k",
-        "swin_L_384_22k",
-    ]
-
-    model_para_dict = {
-        "swin_T_224_1k": dict(
-            embed_dim=96, depths=[2, 2, 6, 2], num_heads=[3, 6, 12, 24], window_size=7
-        ),
-        "swin_B_224_22k": dict(
-            embed_dim=128, depths=[2, 2, 18, 2], num_heads=[4, 8, 16, 32], window_size=7
-        ),
-        "swin_B_384_22k": dict(
-            embed_dim=128, depths=[2, 2, 18, 2], num_heads=[4, 8, 16, 32], window_size=12
-        ),
-        "swin_L_224_22k": dict(
-            embed_dim=192, depths=[2, 2, 18, 2], num_heads=[6, 12, 24, 48], window_size=7
-        ),
-        "swin_L_384_22k": dict(
-            embed_dim=192, depths=[2, 2, 18, 2], num_heads=[6, 12, 24, 48], window_size=12
-        ),
-    }
-    kw_cgf = model_para_dict[modelname]
-    kw_cgf.update(kw)
-    model = SwinTransformer(pretrain_img_size=pretrain_img_size, **kw_cgf)
-    return model
-
-
-if __name__ == "__main__":
-    model = build_swin_transformer("swin_L_384_22k", 384, dilation=True)
-    x = torch.rand(2, 3, 1024, 1024)
-    y = model.forward_raw(x)
-    import ipdb
-
-    ipdb.set_trace()
-    x = torch.rand(2, 3, 384, 384)
-    y = model.forward_raw(x)

grounding_dino/groundingdino/models/GroundingDINO/bertwarper.py

@@ -1,273 +0,0 @@
-# ------------------------------------------------------------------------
-# Grounding DINO
-# url: https://github.com/IDEA-Research/GroundingDINO
-# Copyright (c) 2023 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-
-import torch
-import torch.nn.functional as F
-import torch.utils.checkpoint as checkpoint
-from torch import Tensor, nn
-from torchvision.ops.boxes import nms
-from transformers import BertConfig, BertModel, BertPreTrainedModel
-from transformers.modeling_outputs import BaseModelOutputWithPoolingAndCrossAttentions
-
-
-class BertModelWarper(nn.Module):
-    def __init__(self, bert_model):
-        super().__init__()
-        # self.bert = bert_modelc
-
-        self.config = bert_model.config
-        self.embeddings = bert_model.embeddings
-        self.encoder = bert_model.encoder
-        self.pooler = bert_model.pooler
-
-        self.get_extended_attention_mask = bert_model.get_extended_attention_mask
-        self.invert_attention_mask = bert_model.invert_attention_mask
-        self.get_head_mask = bert_model.get_head_mask
-
-    def forward(
-        self,
-        input_ids=None,
-        attention_mask=None,
-        token_type_ids=None,
-        position_ids=None,
-        head_mask=None,
-        inputs_embeds=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
-        past_key_values=None,
-        use_cache=None,
-        output_attentions=None,
-        output_hidden_states=None,
-        return_dict=None,
-    ):
-        r"""
-        encoder_hidden_states  (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`):
-            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
-            the model is configured as a decoder.
-        encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
-            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
-            the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``:
-
-            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
-        past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
-            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
-
-            If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids`
-            (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)`
-            instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`.
-        use_cache (:obj:`bool`, `optional`):
-            If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up
-            decoding (see :obj:`past_key_values`).
-        """
-        output_attentions = (
-            output_attentions if output_attentions is not None else self.config.output_attentions
-        )
-        output_hidden_states = (
-            output_hidden_states
-            if output_hidden_states is not None
-            else self.config.output_hidden_states
-        )
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
-        if self.config.is_decoder:
-            use_cache = use_cache if use_cache is not None else self.config.use_cache
-        else:
-            use_cache = False
-
-        if input_ids is not None and inputs_embeds is not None:
-            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
-        elif input_ids is not None:
-            input_shape = input_ids.size()
-            batch_size, seq_length = input_shape
-        elif inputs_embeds is not None:
-            input_shape = inputs_embeds.size()[:-1]
-            batch_size, seq_length = input_shape
-        else:
-            raise ValueError("You have to specify either input_ids or inputs_embeds")
-
-        device = input_ids.device if input_ids is not None else inputs_embeds.device
-
-        # past_key_values_length
-        past_key_values_length = (
-            past_key_values[0][0].shape[2] if past_key_values is not None else 0
-        )
-
-        if attention_mask is None:
-            attention_mask = torch.ones(
-                ((batch_size, seq_length + past_key_values_length)), device=device
-            )
-        if token_type_ids is None:
-            token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device)
-
-        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
-        # ourselves in which case we just need to make it broadcastable to all heads.
-        extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(
-            attention_mask, input_shape, device
-        )
-
-        # If a 2D or 3D attention mask is provided for the cross-attention
-        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
-        if self.config.is_decoder and encoder_hidden_states is not None:
-            encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()
-            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
-            if encoder_attention_mask is None:
-                encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
-            encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
-        else:
-            encoder_extended_attention_mask = None
-        # if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO':
-        #     import ipdb; ipdb.set_trace()
-
-        # Prepare head mask if needed
-        # 1.0 in head_mask indicate we keep the head
-        # attention_probs has shape bsz x n_heads x N x N
-        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
-        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
-        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
-
-        embedding_output = self.embeddings(
-            input_ids=input_ids,
-            position_ids=position_ids,
-            token_type_ids=token_type_ids,
-            inputs_embeds=inputs_embeds,
-            past_key_values_length=past_key_values_length,
-        )
-
-        encoder_outputs = self.encoder(
-            embedding_output,
-            attention_mask=extended_attention_mask,
-            head_mask=head_mask,
-            encoder_hidden_states=encoder_hidden_states,
-            encoder_attention_mask=encoder_extended_attention_mask,
-            past_key_values=past_key_values,
-            use_cache=use_cache,
-            output_attentions=output_attentions,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-        )
-        sequence_output = encoder_outputs[0]
-        pooled_output = self.pooler(sequence_output) if self.pooler is not None else None
-
-        if not return_dict:
-            return (sequence_output, pooled_output) + encoder_outputs[1:]
-
-        return BaseModelOutputWithPoolingAndCrossAttentions(
-            last_hidden_state=sequence_output,
-            pooler_output=pooled_output,
-            past_key_values=encoder_outputs.past_key_values,
-            hidden_states=encoder_outputs.hidden_states,
-            attentions=encoder_outputs.attentions,
-            cross_attentions=encoder_outputs.cross_attentions,
-        )
-
-
-class TextEncoderShell(nn.Module):
-    def __init__(self, text_encoder):
-        super().__init__()
-        self.text_encoder = text_encoder
-        self.config = self.text_encoder.config
-
-    def forward(self, **kw):
-        # feed into text encoder
-        return self.text_encoder(**kw)
-
-
-def generate_masks_with_special_tokens(tokenized, special_tokens_list, tokenizer):
-    """Generate attention mask between each pair of special tokens
-    Args:
-        input_ids (torch.Tensor): input ids. Shape: [bs, num_token]
-        special_tokens_mask (list): special tokens mask.
-    Returns:
-        torch.Tensor: attention mask between each special tokens.
-    """
-    input_ids = tokenized["input_ids"]
-    bs, num_token = input_ids.shape
-    # special_tokens_mask: bs, num_token. 1 for special tokens. 0 for normal tokens
-    special_tokens_mask = torch.zeros((bs, num_token), device=input_ids.device).bool()
-    for special_token in special_tokens_list:
-        special_tokens_mask |= input_ids == special_token
-
-    # idxs: each row is a list of indices of special tokens
-    idxs = torch.nonzero(special_tokens_mask)
-
-    # generate attention mask and positional ids
-    attention_mask = (
-        torch.eye(num_token, device=input_ids.device).bool().unsqueeze(0).repeat(bs, 1, 1)
-    )
-    position_ids = torch.zeros((bs, num_token), device=input_ids.device)
-    previous_col = 0
-    for i in range(idxs.shape[0]):
-        row, col = idxs[i]
-        if (col == 0) or (col == num_token - 1):
-            attention_mask[row, col, col] = True
-            position_ids[row, col] = 0
-        else:
-            attention_mask[row, previous_col + 1 : col + 1, previous_col + 1 : col + 1] = True
-            position_ids[row, previous_col + 1 : col + 1] = torch.arange(
-                0, col - previous_col, device=input_ids.device
-            )
-
-        previous_col = col
-
-    # # padding mask
-    # padding_mask = tokenized['attention_mask']
-    # attention_mask = attention_mask & padding_mask.unsqueeze(1).bool() & padding_mask.unsqueeze(2).bool()
-
-    return attention_mask, position_ids.to(torch.long)
-
-
-def generate_masks_with_special_tokens_and_transfer_map(tokenized, special_tokens_list, tokenizer):
-    """Generate attention mask between each pair of special tokens
-    Args:
-        input_ids (torch.Tensor): input ids. Shape: [bs, num_token]
-        special_tokens_mask (list): special tokens mask.
-    Returns:
-        torch.Tensor: attention mask between each special tokens.
-    """
-    input_ids = tokenized["input_ids"]
-    bs, num_token = input_ids.shape
-    # special_tokens_mask: bs, num_token. 1 for special tokens. 0 for normal tokens
-    special_tokens_mask = torch.zeros((bs, num_token), device=input_ids.device).bool()
-    for special_token in special_tokens_list:
-        special_tokens_mask |= input_ids == special_token
-
-    # idxs: each row is a list of indices of special tokens
-    idxs = torch.nonzero(special_tokens_mask)
-
-    # generate attention mask and positional ids
-    attention_mask = (
-        torch.eye(num_token, device=input_ids.device).bool().unsqueeze(0).repeat(bs, 1, 1)
-    )
-    position_ids = torch.zeros((bs, num_token), device=input_ids.device)
-    cate_to_token_mask_list = [[] for _ in range(bs)]
-    previous_col = 0
-    for i in range(idxs.shape[0]):
-        row, col = idxs[i]
-        if (col == 0) or (col == num_token - 1):
-            attention_mask[row, col, col] = True
-            position_ids[row, col] = 0
-        else:
-            attention_mask[row, previous_col + 1 : col + 1, previous_col + 1 : col + 1] = True
-            position_ids[row, previous_col + 1 : col + 1] = torch.arange(
-                0, col - previous_col, device=input_ids.device
-            )
-            c2t_maski = torch.zeros((num_token), device=input_ids.device).bool()
-            c2t_maski[previous_col + 1 : col] = True
-            cate_to_token_mask_list[row].append(c2t_maski)
-        previous_col = col
-
-    cate_to_token_mask_list = [
-        torch.stack(cate_to_token_mask_listi, dim=0)
-        for cate_to_token_mask_listi in cate_to_token_mask_list
-    ]
-
-    # # padding mask
-    # padding_mask = tokenized['attention_mask']
-    # attention_mask = attention_mask & padding_mask.unsqueeze(1).bool() & padding_mask.unsqueeze(2).bool()
-
-    return attention_mask, position_ids.to(torch.long), cate_to_token_mask_list

grounding_dino/groundingdino/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn.h

@@ -1,64 +0,0 @@
-/*!
-**************************************************************************************************
-* Deformable DETR
-* Copyright (c) 2020 SenseTime. All Rights Reserved.
-* Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-**************************************************************************************************
-* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0
-**************************************************************************************************
-*/
-
-#pragma once
-
-#include "ms_deform_attn_cpu.h"
-
-#ifdef WITH_CUDA
-#include "ms_deform_attn_cuda.h"
-#endif
-
-namespace groundingdino {
-
-at::Tensor
-ms_deform_attn_forward(
-    const at::Tensor &value, 
-    const at::Tensor &spatial_shapes,
-    const at::Tensor &level_start_index,
-    const at::Tensor &sampling_loc,
-    const at::Tensor &attn_weight,
-    const int im2col_step)
-{
-    if (value.type().is_cuda())
-    {
-#ifdef WITH_CUDA
-        return ms_deform_attn_cuda_forward(
-            value, spatial_shapes, level_start_index, sampling_loc, attn_weight, im2col_step);
-#else
-        AT_ERROR("Not compiled with GPU support");
-#endif
-    }
-    AT_ERROR("Not implemented on the CPU");
-}
-
-std::vector<at::Tensor>
-ms_deform_attn_backward(
-    const at::Tensor &value, 
-    const at::Tensor &spatial_shapes,
-    const at::Tensor &level_start_index,
-    const at::Tensor &sampling_loc,
-    const at::Tensor &attn_weight,
-    const at::Tensor &grad_output,
-    const int im2col_step)
-{
-    if (value.type().is_cuda())
-    {
-#ifdef WITH_CUDA
-        return ms_deform_attn_cuda_backward(
-            value, spatial_shapes, level_start_index, sampling_loc, attn_weight, grad_output, im2col_step);
-#else
-        AT_ERROR("Not compiled with GPU support");
-#endif
-    }
-    AT_ERROR("Not implemented on the CPU");
-}
-
-} // namespace groundingdino

grounding_dino/groundingdino/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cpu.cpp

@@ -1,43 +0,0 @@
-/*!
-**************************************************************************************************
-* Deformable DETR
-* Copyright (c) 2020 SenseTime. All Rights Reserved.
-* Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-**************************************************************************************************
-* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0
-**************************************************************************************************
-*/
-
-#include <vector>
-
-#include <ATen/ATen.h>
-#include <ATen/cuda/CUDAContext.h>
-
-namespace groundingdino {
-
-at::Tensor
-ms_deform_attn_cpu_forward(
-    const at::Tensor &value, 
-    const at::Tensor &spatial_shapes,
-    const at::Tensor &level_start_index,
-    const at::Tensor &sampling_loc,
-    const at::Tensor &attn_weight,
-    const int im2col_step)
-{
-    AT_ERROR("Not implement on cpu");
-}
-
-std::vector<at::Tensor>
-ms_deform_attn_cpu_backward(
-    const at::Tensor &value, 
-    const at::Tensor &spatial_shapes,
-    const at::Tensor &level_start_index,
-    const at::Tensor &sampling_loc,
-    const at::Tensor &attn_weight,
-    const at::Tensor &grad_output,
-    const int im2col_step)
-{
-    AT_ERROR("Not implement on cpu");
-}
-
-} // namespace groundingdino

grounding_dino/groundingdino/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cpu.h

@@ -1,35 +0,0 @@
-/*!
-**************************************************************************************************
-* Deformable DETR
-* Copyright (c) 2020 SenseTime. All Rights Reserved.
-* Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-**************************************************************************************************
-* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0
-**************************************************************************************************
-*/
-
-#pragma once
-#include <torch/extension.h>
-
-namespace groundingdino {
-
-at::Tensor
-ms_deform_attn_cpu_forward(
-    const at::Tensor &value, 
-    const at::Tensor &spatial_shapes,
-    const at::Tensor &level_start_index,
-    const at::Tensor &sampling_loc,
-    const at::Tensor &attn_weight,
-    const int im2col_step);
-
-std::vector<at::Tensor>
-ms_deform_attn_cpu_backward(
-    const at::Tensor &value, 
-    const at::Tensor &spatial_shapes,
-    const at::Tensor &level_start_index,
-    const at::Tensor &sampling_loc,
-    const at::Tensor &attn_weight,
-    const at::Tensor &grad_output,
-    const int im2col_step);
-
-} // namespace groundingdino

grounding_dino/groundingdino/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cuda.cu

@@ -1,156 +0,0 @@
-/*!
-**************************************************************************************************
-* Deformable DETR
-* Copyright (c) 2020 SenseTime. All Rights Reserved.
-* Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-**************************************************************************************************
-* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0
-**************************************************************************************************
-*/
-
-#include <vector>
-#include "ms_deform_im2col_cuda.cuh"
-
-#include <ATen/ATen.h>
-#include <ATen/cuda/CUDAContext.h>
-#include <cuda.h>
-#include <cuda_runtime.h>
-
-namespace groundingdino {
-
-at::Tensor ms_deform_attn_cuda_forward(
-    const at::Tensor &value, 
-    const at::Tensor &spatial_shapes,
-    const at::Tensor &level_start_index,
-    const at::Tensor &sampling_loc,
-    const at::Tensor &attn_weight,
-    const int im2col_step)
-{
-    AT_ASSERTM(value.is_contiguous(), "value tensor has to be contiguous");
-    AT_ASSERTM(spatial_shapes.is_contiguous(), "spatial_shapes tensor has to be contiguous");
-    AT_ASSERTM(level_start_index.is_contiguous(), "level_start_index tensor has to be contiguous");
-    AT_ASSERTM(sampling_loc.is_contiguous(), "sampling_loc tensor has to be contiguous");
-    AT_ASSERTM(attn_weight.is_contiguous(), "attn_weight tensor has to be contiguous");
-
-    AT_ASSERTM(value.type().is_cuda(), "value must be a CUDA tensor");
-    AT_ASSERTM(spatial_shapes.type().is_cuda(), "spatial_shapes must be a CUDA tensor");
-    AT_ASSERTM(level_start_index.type().is_cuda(), "level_start_index must be a CUDA tensor");
-    AT_ASSERTM(sampling_loc.type().is_cuda(), "sampling_loc must be a CUDA tensor");
-    AT_ASSERTM(attn_weight.type().is_cuda(), "attn_weight must be a CUDA tensor");
-
-    const int batch = value.size(0);
-    const int spatial_size = value.size(1);
-    const int num_heads = value.size(2);
-    const int channels = value.size(3);
-
-    const int num_levels = spatial_shapes.size(0);
-
-    const int num_query = sampling_loc.size(1);
-    const int num_point = sampling_loc.size(4);
-
-    const int im2col_step_ = std::min(batch, im2col_step);
-
-    AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_);
-    
-    auto output = at::zeros({batch, num_query, num_heads, channels}, value.options());
-
-    const int batch_n = im2col_step_;
-    auto output_n = output.view({batch/im2col_step_, batch_n, num_query, num_heads, channels});
-    auto per_value_size = spatial_size * num_heads * channels;
-    auto per_sample_loc_size = num_query * num_heads * num_levels * num_point * 2;
-    auto per_attn_weight_size = num_query * num_heads * num_levels * num_point;
-    for (int n = 0; n < batch/im2col_step_; ++n)
-    {
-        auto columns = output_n.select(0, n);
-        AT_DISPATCH_FLOATING_TYPES(value.type(), "ms_deform_attn_forward_cuda", ([&] {
-            ms_deformable_im2col_cuda(at::cuda::getCurrentCUDAStream(),
-                value.data<scalar_t>() + n * im2col_step_ * per_value_size,
-                spatial_shapes.data<int64_t>(),
-                level_start_index.data<int64_t>(),
-                sampling_loc.data<scalar_t>() + n * im2col_step_ * per_sample_loc_size,
-                attn_weight.data<scalar_t>() + n * im2col_step_ * per_attn_weight_size,
-                batch_n, spatial_size, num_heads, channels, num_levels, num_query, num_point,
-                columns.data<scalar_t>());
-
-        }));
-    }
-
-    output = output.view({batch, num_query, num_heads*channels});
-
-    return output;
-}
-
-
-std::vector<at::Tensor> ms_deform_attn_cuda_backward(
-    const at::Tensor &value, 
-    const at::Tensor &spatial_shapes,
-    const at::Tensor &level_start_index,
-    const at::Tensor &sampling_loc,
-    const at::Tensor &attn_weight,
-    const at::Tensor &grad_output,
-    const int im2col_step)
-{
-
-    AT_ASSERTM(value.is_contiguous(), "value tensor has to be contiguous");
-    AT_ASSERTM(spatial_shapes.is_contiguous(), "spatial_shapes tensor has to be contiguous");
-    AT_ASSERTM(level_start_index.is_contiguous(), "level_start_index tensor has to be contiguous");
-    AT_ASSERTM(sampling_loc.is_contiguous(), "sampling_loc tensor has to be contiguous");
-    AT_ASSERTM(attn_weight.is_contiguous(), "attn_weight tensor has to be contiguous");
-    AT_ASSERTM(grad_output.is_contiguous(), "grad_output tensor has to be contiguous");
-
-    AT_ASSERTM(value.type().is_cuda(), "value must be a CUDA tensor");
-    AT_ASSERTM(spatial_shapes.type().is_cuda(), "spatial_shapes must be a CUDA tensor");
-    AT_ASSERTM(level_start_index.type().is_cuda(), "level_start_index must be a CUDA tensor");
-    AT_ASSERTM(sampling_loc.type().is_cuda(), "sampling_loc must be a CUDA tensor");
-    AT_ASSERTM(attn_weight.type().is_cuda(), "attn_weight must be a CUDA tensor");
-    AT_ASSERTM(grad_output.type().is_cuda(), "grad_output must be a CUDA tensor");
-
-    const int batch = value.size(0);
-    const int spatial_size = value.size(1);
-    const int num_heads = value.size(2);
-    const int channels = value.size(3);
-
-    const int num_levels = spatial_shapes.size(0);
-
-    const int num_query = sampling_loc.size(1);
-    const int num_point = sampling_loc.size(4);
-
-    const int im2col_step_ = std::min(batch, im2col_step);
-
-    AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_);
-
-    auto grad_value = at::zeros_like(value);
-    auto grad_sampling_loc = at::zeros_like(sampling_loc);
-    auto grad_attn_weight = at::zeros_like(attn_weight);
-
-    const int batch_n = im2col_step_;
-    auto per_value_size = spatial_size * num_heads * channels;
-    auto per_sample_loc_size = num_query * num_heads * num_levels * num_point * 2;
-    auto per_attn_weight_size = num_query * num_heads * num_levels * num_point;
-    auto grad_output_n = grad_output.view({batch/im2col_step_, batch_n, num_query, num_heads, channels});
-    
-    for (int n = 0; n < batch/im2col_step_; ++n)
-    {
-        auto grad_output_g = grad_output_n.select(0, n);
-        AT_DISPATCH_FLOATING_TYPES(value.type(), "ms_deform_attn_backward_cuda", ([&] {
-            ms_deformable_col2im_cuda(at::cuda::getCurrentCUDAStream(),
-                                    grad_output_g.data<scalar_t>(),
-                                    value.data<scalar_t>() + n * im2col_step_ * per_value_size,
-                                    spatial_shapes.data<int64_t>(),
-                                    level_start_index.data<int64_t>(),
-                                    sampling_loc.data<scalar_t>() + n * im2col_step_ * per_sample_loc_size,
-                                    attn_weight.data<scalar_t>() + n * im2col_step_ * per_attn_weight_size,
-                                    batch_n, spatial_size, num_heads, channels, num_levels, num_query, num_point,
-                                    grad_value.data<scalar_t>() +  n * im2col_step_ * per_value_size,
-                                    grad_sampling_loc.data<scalar_t>() + n * im2col_step_ * per_sample_loc_size,
-                                    grad_attn_weight.data<scalar_t>() + n * im2col_step_ * per_attn_weight_size);
-
-        }));
-    }
-
-    return {
-        grad_value, grad_sampling_loc, grad_attn_weight
-    };
-}
-
-} // namespace groundingdino

grounding_dino/groundingdino/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cuda.h

@@ -1,33 +0,0 @@
-/*!
-**************************************************************************************************
-* Deformable DETR
-* Copyright (c) 2020 SenseTime. All Rights Reserved.
-* Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-**************************************************************************************************
-* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0
-**************************************************************************************************
-*/
-
-#pragma once
-#include <torch/extension.h>
-
-namespace groundingdino {
-
-at::Tensor ms_deform_attn_cuda_forward(
-    const at::Tensor &value, 
-    const at::Tensor &spatial_shapes,
-    const at::Tensor &level_start_index,
-    const at::Tensor &sampling_loc,
-    const at::Tensor &attn_weight,
-    const int im2col_step);
-
-std::vector<at::Tensor> ms_deform_attn_cuda_backward(
-    const at::Tensor &value, 
-    const at::Tensor &spatial_shapes,
-    const at::Tensor &level_start_index,
-    const at::Tensor &sampling_loc,
-    const at::Tensor &attn_weight,
-    const at::Tensor &grad_output,
-    const int im2col_step);
-
-} // namespace groundingdino

grounding_dino/groundingdino/models/GroundingDINO/csrc/cuda_version.cu

@@ -1,7 +0,0 @@
-#include <cuda_runtime_api.h>
-
-namespace groundingdino {
-int get_cudart_version() {
-  return CUDART_VERSION;
-}
-} // namespace groundingdino

grounding_dino/groundingdino/models/GroundingDINO/csrc/vision.cpp

@@ -1,58 +0,0 @@
-// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-
-#include "MsDeformAttn/ms_deform_attn.h"
-
-namespace groundingdino {
-
-#ifdef WITH_CUDA
-extern int get_cudart_version();
-#endif
-
-std::string get_cuda_version() {
-#ifdef WITH_CUDA
-  std::ostringstream oss;
-
-  // copied from
-  // https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/cuda/detail/CUDAHooks.cpp#L231
-  auto printCudaStyleVersion = [&](int v) {
-    oss << (v / 1000) << "." << (v / 10 % 100);
-    if (v % 10 != 0) {
-      oss << "." << (v % 10);
-    }
-  };
-  printCudaStyleVersion(get_cudart_version());
-  return oss.str();
-#else
-  return std::string("not available");
-#endif
-}
-
-// similar to
-// https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/Version.cpp
-std::string get_compiler_version() {
-  std::ostringstream ss;
-#if defined(__GNUC__)
-#ifndef __clang__
-  { ss << "GCC " << __GNUC__ << "." << __GNUC_MINOR__; }
-#endif
-#endif
-
-#if defined(__clang_major__)
-  {
-    ss << "clang " << __clang_major__ << "." << __clang_minor__ << "."
-       << __clang_patchlevel__;
-  }
-#endif
-
-#if defined(_MSC_VER)
-  { ss << "MSVC " << _MSC_FULL_VER; }
-#endif
-  return ss.str();
-}
-
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
-  m.def("ms_deform_attn_forward", &ms_deform_attn_forward, "ms_deform_attn_forward");
-  m.def("ms_deform_attn_backward", &ms_deform_attn_backward, "ms_deform_attn_backward");
-}
-
-} // namespace groundingdino

grounding_dino/groundingdino/models/GroundingDINO/fuse_modules.py

@@ -1,297 +0,0 @@
-# ------------------------------------------------------------------------
-# Grounding DINO
-# url: https://github.com/IDEA-Research/GroundingDINO
-# Copyright (c) 2023 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from timm.models.layers import DropPath
-
-
-class FeatureResizer(nn.Module):
-    """
-    This class takes as input a set of embeddings of dimension C1 and outputs a set of
-    embedding of dimension C2, after a linear transformation, dropout and normalization (LN).
-    """
-
-    def __init__(self, input_feat_size, output_feat_size, dropout, do_ln=True):
-        super().__init__()
-        self.do_ln = do_ln
-        # Object feature encoding
-        self.fc = nn.Linear(input_feat_size, output_feat_size, bias=True)
-        self.layer_norm = nn.LayerNorm(output_feat_size, eps=1e-12)
-        self.dropout = nn.Dropout(dropout)
-
-    def forward(self, encoder_features):
-        x = self.fc(encoder_features)
-        if self.do_ln:
-            x = self.layer_norm(x)
-        output = self.dropout(x)
-        return output
-
-
-def l1norm(X, dim, eps=1e-8):
-    """L1-normalize columns of X"""
-    norm = torch.abs(X).sum(dim=dim, keepdim=True) + eps
-    X = torch.div(X, norm)
-    return X
-
-
-def l2norm(X, dim, eps=1e-8):
-    """L2-normalize columns of X"""
-    norm = torch.pow(X, 2).sum(dim=dim, keepdim=True).sqrt() + eps
-    X = torch.div(X, norm)
-    return X
-
-
-def func_attention(query, context, smooth=1, raw_feature_norm="softmax", eps=1e-8):
-    """
-    query: (n_context, queryL, d)
-    context: (n_context, sourceL, d)
-    """
-    batch_size_q, queryL = query.size(0), query.size(1)
-    batch_size, sourceL = context.size(0), context.size(1)
-
-    # Get attention
-    # --> (batch, d, queryL)
-    queryT = torch.transpose(query, 1, 2)
-
-    # (batch, sourceL, d)(batch, d, queryL)
-    # --> (batch, sourceL, queryL)
-    attn = torch.bmm(context, queryT)
-    if raw_feature_norm == "softmax":
-        # --> (batch*sourceL, queryL)
-        attn = attn.view(batch_size * sourceL, queryL)
-        attn = nn.Softmax()(attn)
-        # --> (batch, sourceL, queryL)
-        attn = attn.view(batch_size, sourceL, queryL)
-    elif raw_feature_norm == "l2norm":
-        attn = l2norm(attn, 2)
-    elif raw_feature_norm == "clipped_l2norm":
-        attn = nn.LeakyReLU(0.1)(attn)
-        attn = l2norm(attn, 2)
-    else:
-        raise ValueError("unknown first norm type:", raw_feature_norm)
-    # --> (batch, queryL, sourceL)
-    attn = torch.transpose(attn, 1, 2).contiguous()
-    # --> (batch*queryL, sourceL)
-    attn = attn.view(batch_size * queryL, sourceL)
-    attn = nn.Softmax()(attn * smooth)
-    # --> (batch, queryL, sourceL)
-    attn = attn.view(batch_size, queryL, sourceL)
-    # --> (batch, sourceL, queryL)
-    attnT = torch.transpose(attn, 1, 2).contiguous()
-
-    # --> (batch, d, sourceL)
-    contextT = torch.transpose(context, 1, 2)
-    # (batch x d x sourceL)(batch x sourceL x queryL)
-    # --> (batch, d, queryL)
-    weightedContext = torch.bmm(contextT, attnT)
-    # --> (batch, queryL, d)
-    weightedContext = torch.transpose(weightedContext, 1, 2)
-
-    return weightedContext, attnT
-
-
-class BiMultiHeadAttention(nn.Module):
-    def __init__(self, v_dim, l_dim, embed_dim, num_heads, dropout=0.1, cfg=None):
-        super(BiMultiHeadAttention, self).__init__()
-
-        self.embed_dim = embed_dim
-        self.num_heads = num_heads
-        self.head_dim = embed_dim // num_heads
-        self.v_dim = v_dim
-        self.l_dim = l_dim
-
-        assert (
-            self.head_dim * self.num_heads == self.embed_dim
-        ), f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`: {self.num_heads})."
-        self.scale = self.head_dim ** (-0.5)
-        self.dropout = dropout
-
-        self.v_proj = nn.Linear(self.v_dim, self.embed_dim)
-        self.l_proj = nn.Linear(self.l_dim, self.embed_dim)
-        self.values_v_proj = nn.Linear(self.v_dim, self.embed_dim)
-        self.values_l_proj = nn.Linear(self.l_dim, self.embed_dim)
-
-        self.out_v_proj = nn.Linear(self.embed_dim, self.v_dim)
-        self.out_l_proj = nn.Linear(self.embed_dim, self.l_dim)
-
-        self.stable_softmax_2d = True
-        self.clamp_min_for_underflow = True
-        self.clamp_max_for_overflow = True
-
-        self._reset_parameters()
-
-    def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
-        return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
-
-    def _reset_parameters(self):
-        nn.init.xavier_uniform_(self.v_proj.weight)
-        self.v_proj.bias.data.fill_(0)
-        nn.init.xavier_uniform_(self.l_proj.weight)
-        self.l_proj.bias.data.fill_(0)
-        nn.init.xavier_uniform_(self.values_v_proj.weight)
-        self.values_v_proj.bias.data.fill_(0)
-        nn.init.xavier_uniform_(self.values_l_proj.weight)
-        self.values_l_proj.bias.data.fill_(0)
-        nn.init.xavier_uniform_(self.out_v_proj.weight)
-        self.out_v_proj.bias.data.fill_(0)
-        nn.init.xavier_uniform_(self.out_l_proj.weight)
-        self.out_l_proj.bias.data.fill_(0)
-
-    def forward(self, v, l, attention_mask_v=None, attention_mask_l=None):
-        """_summary_
-
-        Args:
-            v (_type_): bs, n_img, dim
-            l (_type_): bs, n_text, dim
-            attention_mask_v (_type_, optional): _description_. bs, n_img
-            attention_mask_l (_type_, optional): _description_. bs, n_text
-
-        Returns:
-            _type_: _description_
-        """
-        # if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO':
-        #     import ipdb; ipdb.set_trace()
-        bsz, tgt_len, _ = v.size()
-
-        query_states = self.v_proj(v) * self.scale
-        key_states = self._shape(self.l_proj(l), -1, bsz)
-        value_v_states = self._shape(self.values_v_proj(v), -1, bsz)
-        value_l_states = self._shape(self.values_l_proj(l), -1, bsz)
-
-        proj_shape = (bsz * self.num_heads, -1, self.head_dim)
-        query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape)
-        key_states = key_states.view(*proj_shape)
-        value_v_states = value_v_states.view(*proj_shape)
-        value_l_states = value_l_states.view(*proj_shape)
-
-        src_len = key_states.size(1)
-        attn_weights = torch.bmm(query_states, key_states.transpose(1, 2))  # bs*nhead, nimg, ntxt
-
-        if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len):
-            raise ValueError(
-                f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is {attn_weights.size()}"
-            )
-
-        if self.stable_softmax_2d:
-            attn_weights = attn_weights - attn_weights.max()
-
-        if self.clamp_min_for_underflow:
-            attn_weights = torch.clamp(
-                attn_weights, min=-50000
-            )  # Do not increase -50000, data type half has quite limited range
-        if self.clamp_max_for_overflow:
-            attn_weights = torch.clamp(
-                attn_weights, max=50000
-            )  # Do not increase 50000, data type half has quite limited range
-
-        attn_weights_T = attn_weights.transpose(1, 2)
-        attn_weights_l = attn_weights_T - torch.max(attn_weights_T, dim=-1, keepdim=True)[0]
-        if self.clamp_min_for_underflow:
-            attn_weights_l = torch.clamp(
-                attn_weights_l, min=-50000
-            )  # Do not increase -50000, data type half has quite limited range
-        if self.clamp_max_for_overflow:
-            attn_weights_l = torch.clamp(
-                attn_weights_l, max=50000
-            )  # Do not increase 50000, data type half has quite limited range
-
-        # mask vison for language
-        if attention_mask_v is not None:
-            attention_mask_v = (
-                attention_mask_v[:, None, None, :].repeat(1, self.num_heads, 1, 1).flatten(0, 1)
-            )
-            attn_weights_l.masked_fill_(attention_mask_v, float("-inf"))
-
-        attn_weights_l = attn_weights_l.softmax(dim=-1)
-
-        # mask language for vision
-        if attention_mask_l is not None:
-            attention_mask_l = (
-                attention_mask_l[:, None, None, :].repeat(1, self.num_heads, 1, 1).flatten(0, 1)
-            )
-            attn_weights.masked_fill_(attention_mask_l, float("-inf"))
-        attn_weights_v = attn_weights.softmax(dim=-1)
-
-        attn_probs_v = F.dropout(attn_weights_v, p=self.dropout, training=self.training)
-        attn_probs_l = F.dropout(attn_weights_l, p=self.dropout, training=self.training)
-
-        attn_output_v = torch.bmm(attn_probs_v, value_l_states)
-        attn_output_l = torch.bmm(attn_probs_l, value_v_states)
-
-        if attn_output_v.size() != (bsz * self.num_heads, tgt_len, self.head_dim):
-            raise ValueError(
-                f"`attn_output_v` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is {attn_output_v.size()}"
-            )
-
-        if attn_output_l.size() != (bsz * self.num_heads, src_len, self.head_dim):
-            raise ValueError(
-                f"`attn_output_l` should be of size {(bsz, self.num_heads, src_len, self.head_dim)}, but is {attn_output_l.size()}"
-            )
-
-        attn_output_v = attn_output_v.view(bsz, self.num_heads, tgt_len, self.head_dim)
-        attn_output_v = attn_output_v.transpose(1, 2)
-        attn_output_v = attn_output_v.reshape(bsz, tgt_len, self.embed_dim)
-
-        attn_output_l = attn_output_l.view(bsz, self.num_heads, src_len, self.head_dim)
-        attn_output_l = attn_output_l.transpose(1, 2)
-        attn_output_l = attn_output_l.reshape(bsz, src_len, self.embed_dim)
-
-        attn_output_v = self.out_v_proj(attn_output_v)
-        attn_output_l = self.out_l_proj(attn_output_l)
-
-        return attn_output_v, attn_output_l
-
-
-# Bi-Direction MHA (text->image, image->text)
-class BiAttentionBlock(nn.Module):
-    def __init__(
-        self,
-        v_dim,
-        l_dim,
-        embed_dim,
-        num_heads,
-        dropout=0.1,
-        drop_path=0.0,
-        init_values=1e-4,
-        cfg=None,
-    ):
-        """
-        Inputs:
-            embed_dim - Dimensionality of input and attention feature vectors
-            hidden_dim - Dimensionality of hidden layer in feed-forward network
-                         (usually 2-4x larger than embed_dim)
-            num_heads - Number of heads to use in the Multi-Head Attention block
-            dropout - Amount of dropout to apply in the feed-forward network
-        """
-        super(BiAttentionBlock, self).__init__()
-
-        # pre layer norm
-        self.layer_norm_v = nn.LayerNorm(v_dim)
-        self.layer_norm_l = nn.LayerNorm(l_dim)
-        self.attn = BiMultiHeadAttention(
-            v_dim=v_dim, l_dim=l_dim, embed_dim=embed_dim, num_heads=num_heads, dropout=dropout
-        )
-
-        # add layer scale for training stability
-        self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
-        self.gamma_v = nn.Parameter(init_values * torch.ones((v_dim)), requires_grad=True)
-        self.gamma_l = nn.Parameter(init_values * torch.ones((l_dim)), requires_grad=True)
-
-    def forward(self, v, l, attention_mask_v=None, attention_mask_l=None):
-        v = self.layer_norm_v(v)
-        l = self.layer_norm_l(l)
-        delta_v, delta_l = self.attn(
-            v, l, attention_mask_v=attention_mask_v, attention_mask_l=attention_mask_l
-        )
-        # v, l = v + delta_v, l + delta_l
-        v = v + self.drop_path(self.gamma_v * delta_v)
-        l = l + self.drop_path(self.gamma_l * delta_l)
-        return v, l
-
-    # def forward(self, v:List[torch.Tensor], l, attention_mask_v=None, attention_mask_l=None)

grounding_dino/groundingdino/models/GroundingDINO/groundingdino.py

@@ -1,412 +0,0 @@
-# ------------------------------------------------------------------------
-# Grounding DINO
-# url: https://github.com/IDEA-Research/GroundingDINO
-# Copyright (c) 2023 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# Conditional DETR model and criterion classes.
-# Copyright (c) 2021 Microsoft. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# Modified from DETR (https://github.com/facebookresearch/detr)
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-# ------------------------------------------------------------------------
-# Modified from Deformable DETR (https://github.com/fundamentalvision/Deformable-DETR)
-# Copyright (c) 2020 SenseTime. All Rights Reserved.
-# ------------------------------------------------------------------------
-import copy
-from typing import List
-
-import torch
-import torch.nn.functional as F
-from torch import nn
-from torchvision.ops.boxes import nms
-from transformers import AutoTokenizer, BertModel, BertTokenizer, RobertaModel, RobertaTokenizerFast
-
-from grounding_dino.groundingdino.util import box_ops, get_tokenlizer
-from grounding_dino.groundingdino.util.misc import (
-    NestedTensor,
-    accuracy,
-    get_world_size,
-    interpolate,
-    inverse_sigmoid,
-    is_dist_avail_and_initialized,
-    nested_tensor_from_tensor_list,
-)
-from grounding_dino.groundingdino.util.utils import get_phrases_from_posmap
-from grounding_dino.groundingdino.util.visualizer import COCOVisualizer
-from grounding_dino.groundingdino.util.vl_utils import create_positive_map_from_span
-
-from ..registry import MODULE_BUILD_FUNCS
-from .backbone import build_backbone
-from .bertwarper import (
-    BertModelWarper,
-    generate_masks_with_special_tokens,
-    generate_masks_with_special_tokens_and_transfer_map,
-)
-from .transformer import build_transformer
-from .utils import MLP, ContrastiveEmbed, sigmoid_focal_loss
-
-
-class GroundingDINO(nn.Module):
-    """This is the Cross-Attention Detector module that performs object detection"""
-
-    def __init__(
-        self,
-        backbone,
-        transformer,
-        num_queries,
-        aux_loss=False,
-        iter_update=False,
-        query_dim=2,
-        num_feature_levels=1,
-        nheads=8,
-        # two stage
-        two_stage_type="no",  # ['no', 'standard']
-        dec_pred_bbox_embed_share=True,
-        two_stage_class_embed_share=True,
-        two_stage_bbox_embed_share=True,
-        num_patterns=0,
-        dn_number=100,
-        dn_box_noise_scale=0.4,
-        dn_label_noise_ratio=0.5,
-        dn_labelbook_size=100,
-        text_encoder_type="bert-base-uncased",
-        sub_sentence_present=True,
-        max_text_len=256,
-    ):
-        """Initializes the model.
-        Parameters:
-            backbone: torch module of the backbone to be used. See backbone.py
-            transformer: torch module of the transformer architecture. See transformer.py
-            num_queries: number of object queries, ie detection slot. This is the maximal number of objects
-                         Conditional DETR can detect in a single image. For COCO, we recommend 100 queries.
-            aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used.
-        """
-        super().__init__()
-        self.num_queries = num_queries
-        self.transformer = transformer
-        self.hidden_dim = hidden_dim = transformer.d_model
-        self.num_feature_levels = num_feature_levels
-        self.nheads = nheads
-        self.max_text_len = 256
-        self.sub_sentence_present = sub_sentence_present
-
-        # setting query dim
-        self.query_dim = query_dim
-        assert query_dim == 4
-
-        # for dn training
-        self.num_patterns = num_patterns
-        self.dn_number = dn_number
-        self.dn_box_noise_scale = dn_box_noise_scale
-        self.dn_label_noise_ratio = dn_label_noise_ratio
-        self.dn_labelbook_size = dn_labelbook_size
-
-        # bert
-        self.tokenizer = get_tokenlizer.get_tokenlizer(text_encoder_type)
-        self.bert = get_tokenlizer.get_pretrained_language_model(text_encoder_type)
-        self.bert.pooler.dense.weight.requires_grad_(False)
-        self.bert.pooler.dense.bias.requires_grad_(False)
-        self.bert = BertModelWarper(bert_model=self.bert)
-
-        self.feat_map = nn.Linear(self.bert.config.hidden_size, self.hidden_dim, bias=True)
-        nn.init.constant_(self.feat_map.bias.data, 0)
-        nn.init.xavier_uniform_(self.feat_map.weight.data)
-        # freeze
-
-        # special tokens
-        self.specical_tokens = self.tokenizer.convert_tokens_to_ids(["[CLS]", "[SEP]", ".", "?"])
-
-        # prepare input projection layers
-        if num_feature_levels > 1:
-            num_backbone_outs = len(backbone.num_channels)
-            input_proj_list = []
-            for _ in range(num_backbone_outs):
-                in_channels = backbone.num_channels[_]
-                input_proj_list.append(
-                    nn.Sequential(
-                        nn.Conv2d(in_channels, hidden_dim, kernel_size=1),
-                        nn.GroupNorm(32, hidden_dim),
-                    )
-                )
-            for _ in range(num_feature_levels - num_backbone_outs):
-                input_proj_list.append(
-                    nn.Sequential(
-                        nn.Conv2d(in_channels, hidden_dim, kernel_size=3, stride=2, padding=1),
-                        nn.GroupNorm(32, hidden_dim),
-                    )
-                )
-                in_channels = hidden_dim
-            self.input_proj = nn.ModuleList(input_proj_list)
-        else:
-            assert two_stage_type == "no", "two_stage_type should be no if num_feature_levels=1 !!!"
-            self.input_proj = nn.ModuleList(
-                [
-                    nn.Sequential(
-                        nn.Conv2d(backbone.num_channels[-1], hidden_dim, kernel_size=1),
-                        nn.GroupNorm(32, hidden_dim),
-                    )
-                ]
-            )
-
-        self.backbone = backbone
-        self.aux_loss = aux_loss
-        self.box_pred_damping = box_pred_damping = None
-
-        self.iter_update = iter_update
-        assert iter_update, "Why not iter_update?"
-
-        # prepare pred layers
-        self.dec_pred_bbox_embed_share = dec_pred_bbox_embed_share
-        # prepare class & box embed
-        _class_embed = ContrastiveEmbed()
-
-        _bbox_embed = MLP(hidden_dim, hidden_dim, 4, 3)
-        nn.init.constant_(_bbox_embed.layers[-1].weight.data, 0)
-        nn.init.constant_(_bbox_embed.layers[-1].bias.data, 0)
-
-        if dec_pred_bbox_embed_share:
-            box_embed_layerlist = [_bbox_embed for i in range(transformer.num_decoder_layers)]
-        else:
-            box_embed_layerlist = [
-                copy.deepcopy(_bbox_embed) for i in range(transformer.num_decoder_layers)
-            ]
-        class_embed_layerlist = [_class_embed for i in range(transformer.num_decoder_layers)]
-        self.bbox_embed = nn.ModuleList(box_embed_layerlist)
-        self.class_embed = nn.ModuleList(class_embed_layerlist)
-        self.transformer.decoder.bbox_embed = self.bbox_embed
-        self.transformer.decoder.class_embed = self.class_embed
-
-        # two stage
-        self.two_stage_type = two_stage_type
-        assert two_stage_type in ["no", "standard"], "unknown param {} of two_stage_type".format(
-            two_stage_type
-        )
-        if two_stage_type != "no":
-            if two_stage_bbox_embed_share:
-                assert dec_pred_bbox_embed_share
-                self.transformer.enc_out_bbox_embed = _bbox_embed
-            else:
-                self.transformer.enc_out_bbox_embed = copy.deepcopy(_bbox_embed)
-
-            if two_stage_class_embed_share:
-                assert dec_pred_bbox_embed_share
-                self.transformer.enc_out_class_embed = _class_embed
-            else:
-                self.transformer.enc_out_class_embed = copy.deepcopy(_class_embed)
-
-            self.refpoint_embed = None
-
-        self._reset_parameters()
-
-    def _reset_parameters(self):
-        # init input_proj
-        for proj in self.input_proj:
-            nn.init.xavier_uniform_(proj[0].weight, gain=1)
-            nn.init.constant_(proj[0].bias, 0)
-
-    def set_image_tensor(self, samples: NestedTensor):
-        if isinstance(samples, (list, torch.Tensor)):
-            samples = nested_tensor_from_tensor_list(samples)
-        self.features, self.poss = self.backbone(samples)
-
-    def unset_image_tensor(self):
-        if hasattr(self, 'features'):
-            del self.features
-        if hasattr(self,'poss'):
-            del self.poss 
-
-    def set_image_features(self, features , poss):
-        self.features = features
-        self.poss = poss
-
-    def init_ref_points(self, use_num_queries):
-        self.refpoint_embed = nn.Embedding(use_num_queries, self.query_dim)
-
-    def forward(self, samples: NestedTensor, targets: List = None, **kw):
-        """The forward expects a NestedTensor, which consists of:
-           - samples.tensor: batched images, of shape [batch_size x 3 x H x W]
-           - samples.mask: a binary mask of shape [batch_size x H x W], containing 1 on padded pixels
-
-        It returns a dict with the following elements:
-           - "pred_logits": the classification logits (including no-object) for all queries.
-                            Shape= [batch_size x num_queries x num_classes]
-           - "pred_boxes": The normalized boxes coordinates for all queries, represented as
-                           (center_x, center_y, width, height). These values are normalized in [0, 1],
-                           relative to the size of each individual image (disregarding possible padding).
-                           See PostProcess for information on how to retrieve the unnormalized bounding box.
-           - "aux_outputs": Optional, only returned when auxilary losses are activated. It is a list of
-                            dictionnaries containing the two above keys for each decoder layer.
-        """
-        if targets is None:
-            captions = kw["captions"]
-        else:
-            captions = [t["caption"] for t in targets]
-
-        # encoder texts
-        tokenized = self.tokenizer(captions, padding="longest", return_tensors="pt").to(
-            samples.device
-        )
-        (
-            text_self_attention_masks,
-            position_ids,
-            cate_to_token_mask_list,
-        ) = generate_masks_with_special_tokens_and_transfer_map(
-            tokenized, self.specical_tokens, self.tokenizer
-        )
-
-        if text_self_attention_masks.shape[1] > self.max_text_len:
-            text_self_attention_masks = text_self_attention_masks[
-                :, : self.max_text_len, : self.max_text_len
-            ]
-            position_ids = position_ids[:, : self.max_text_len]
-            tokenized["input_ids"] = tokenized["input_ids"][:, : self.max_text_len]
-            tokenized["attention_mask"] = tokenized["attention_mask"][:, : self.max_text_len]
-            tokenized["token_type_ids"] = tokenized["token_type_ids"][:, : self.max_text_len]
-
-        # extract text embeddings
-        if self.sub_sentence_present:
-            tokenized_for_encoder = {k: v for k, v in tokenized.items() if k != "attention_mask"}
-            tokenized_for_encoder["attention_mask"] = text_self_attention_masks
-            tokenized_for_encoder["position_ids"] = position_ids
-        else:
-            # import ipdb; ipdb.set_trace()
-            tokenized_for_encoder = tokenized
-
-        bert_output = self.bert(**tokenized_for_encoder)  # bs, 195, 768
-
-        encoded_text = self.feat_map(bert_output["last_hidden_state"])  # bs, 195, d_model
-        text_token_mask = tokenized.attention_mask.bool()  # bs, 195
-        # text_token_mask: True for nomask, False for mask
-        # text_self_attention_masks: True for nomask, False for mask
-
-        if encoded_text.shape[1] > self.max_text_len:
-            encoded_text = encoded_text[:, : self.max_text_len, :]
-            text_token_mask = text_token_mask[:, : self.max_text_len]
-            position_ids = position_ids[:, : self.max_text_len]
-            text_self_attention_masks = text_self_attention_masks[
-                :, : self.max_text_len, : self.max_text_len
-            ]
-
-        text_dict = {
-            "encoded_text": encoded_text,  # bs, 195, d_model
-            "text_token_mask": text_token_mask,  # bs, 195
-            "position_ids": position_ids,  # bs, 195
-            "text_self_attention_masks": text_self_attention_masks,  # bs, 195,195
-        }
-
-        # import ipdb; ipdb.set_trace()
-        if isinstance(samples, (list, torch.Tensor)):
-            samples = nested_tensor_from_tensor_list(samples)
-        if not hasattr(self, 'features') or not hasattr(self, 'poss'):
-            self.set_image_tensor(samples)
-
-        srcs = []
-        masks = []
-        for l, feat in enumerate(self.features):
-            src, mask = feat.decompose()
-            srcs.append(self.input_proj[l](src))
-            masks.append(mask)
-            assert mask is not None
-        if self.num_feature_levels > len(srcs):
-            _len_srcs = len(srcs)
-            for l in range(_len_srcs, self.num_feature_levels):
-                if l == _len_srcs:
-                    src = self.input_proj[l](self.features[-1].tensors)
-                else:
-                    src = self.input_proj[l](srcs[-1])
-                m = samples.mask
-                mask = F.interpolate(m[None].float(), size=src.shape[-2:]).to(torch.bool)[0]
-                pos_l = self.backbone[1](NestedTensor(src, mask)).to(src.dtype)
-                srcs.append(src)
-                masks.append(mask)
-                self.poss.append(pos_l)
-
-        input_query_bbox = input_query_label = attn_mask = dn_meta = None
-        hs, reference, hs_enc, ref_enc, init_box_proposal = self.transformer(
-            srcs, masks, input_query_bbox, self.poss, input_query_label, attn_mask, text_dict
-        )
-
-        # deformable-detr-like anchor update
-        outputs_coord_list = []
-        for dec_lid, (layer_ref_sig, layer_bbox_embed, layer_hs) in enumerate(
-            zip(reference[:-1], self.bbox_embed, hs)
-        ):
-            layer_delta_unsig = layer_bbox_embed(layer_hs)
-            layer_outputs_unsig = layer_delta_unsig + inverse_sigmoid(layer_ref_sig)
-            layer_outputs_unsig = layer_outputs_unsig.sigmoid()
-            outputs_coord_list.append(layer_outputs_unsig)
-        outputs_coord_list = torch.stack(outputs_coord_list)
-
-        # output
-        outputs_class = torch.stack(
-            [
-                layer_cls_embed(layer_hs, text_dict)
-                for layer_cls_embed, layer_hs in zip(self.class_embed, hs)
-            ]
-        )
-        out = {"pred_logits": outputs_class[-1], "pred_boxes": outputs_coord_list[-1]}
-
-        # # for intermediate outputs
-        # if self.aux_loss:
-        #     out['aux_outputs'] = self._set_aux_loss(outputs_class, outputs_coord_list)
-
-        # # for encoder output
-        # if hs_enc is not None:
-        #     # prepare intermediate outputs
-        #     interm_coord = ref_enc[-1]
-        #     interm_class = self.transformer.enc_out_class_embed(hs_enc[-1], text_dict)
-        #     out['interm_outputs'] = {'pred_logits': interm_class, 'pred_boxes': interm_coord}
-        #     out['interm_outputs_for_matching_pre'] = {'pred_logits': interm_class, 'pred_boxes': init_box_proposal}
-        unset_image_tensor = kw.get('unset_image_tensor', True)
-        if unset_image_tensor:
-            self.unset_image_tensor() ## If necessary
-        return out
-
-    @torch.jit.unused
-    def _set_aux_loss(self, outputs_class, outputs_coord):
-        # this is a workaround to make torchscript happy, as torchscript
-        # doesn't support dictionary with non-homogeneous values, such
-        # as a dict having both a Tensor and a list.
-        return [
-            {"pred_logits": a, "pred_boxes": b}
-            for a, b in zip(outputs_class[:-1], outputs_coord[:-1])
-        ]
-
-
-@MODULE_BUILD_FUNCS.registe_with_name(module_name="groundingdino")
-def build_groundingdino(args):
-
-    backbone = build_backbone(args)
-    transformer = build_transformer(args)
-
-    dn_labelbook_size = args.dn_labelbook_size
-    dec_pred_bbox_embed_share = args.dec_pred_bbox_embed_share
-    sub_sentence_present = args.sub_sentence_present
-
-    model = GroundingDINO(
-        backbone,
-        transformer,
-        num_queries=args.num_queries,
-        aux_loss=True,
-        iter_update=True,
-        query_dim=4,
-        num_feature_levels=args.num_feature_levels,
-        nheads=args.nheads,
-        dec_pred_bbox_embed_share=dec_pred_bbox_embed_share,
-        two_stage_type=args.two_stage_type,
-        two_stage_bbox_embed_share=args.two_stage_bbox_embed_share,
-        two_stage_class_embed_share=args.two_stage_class_embed_share,
-        num_patterns=args.num_patterns,
-        dn_number=0,
-        dn_box_noise_scale=args.dn_box_noise_scale,
-        dn_label_noise_ratio=args.dn_label_noise_ratio,
-        dn_labelbook_size=dn_labelbook_size,
-        text_encoder_type=args.text_encoder_type,
-        sub_sentence_present=sub_sentence_present,
-        max_text_len=args.max_text_len,
-    )
-
-    return model
-

grounding_dino/groundingdino/models/GroundingDINO/ms_deform_attn.py

@@ -1,413 +0,0 @@
-# ------------------------------------------------------------------------
-# Grounding DINO
-# url: https://github.com/IDEA-Research/GroundingDINO
-# Copyright (c) 2023 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# Deformable DETR
-# Copyright (c) 2020 SenseTime. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------------------------------
-# Modified from:
-# https://github.com/fundamentalvision/Deformable-DETR/blob/main/models/ops/functions/ms_deform_attn_func.py
-# https://github.com/fundamentalvision/Deformable-DETR/blob/main/models/ops/modules/ms_deform_attn.py
-# https://github.com/open-mmlab/mmcv/blob/master/mmcv/ops/multi_scale_deform_attn.py
-# ------------------------------------------------------------------------------------------------
-
-import math
-import warnings
-from typing import Optional
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from torch.autograd import Function
-from torch.autograd.function import once_differentiable
-from torch.nn.init import constant_, xavier_uniform_
-
-try:
-    from grounding_dino.groundingdino import _C
-except:
-    warnings.warn("Failed to load custom C++ ops. Running on CPU mode Only!")
-
-
-# helpers
-def _is_power_of_2(n):
-    if (not isinstance(n, int)) or (n < 0):
-        raise ValueError("invalid input for _is_power_of_2: {} (type: {})".format(n, type(n)))
-    return (n & (n - 1) == 0) and n != 0
-
-
-class MultiScaleDeformableAttnFunction(Function):
-    @staticmethod
-    def forward(
-        ctx,
-        value,
-        value_spatial_shapes,
-        value_level_start_index,
-        sampling_locations,
-        attention_weights,
-        im2col_step,
-    ):
-        ctx.im2col_step = im2col_step
-        output = _C.ms_deform_attn_forward(
-            value,
-            value_spatial_shapes,
-            value_level_start_index,
-            sampling_locations,
-            attention_weights,
-            ctx.im2col_step,
-        )
-        ctx.save_for_backward(
-            value,
-            value_spatial_shapes,
-            value_level_start_index,
-            sampling_locations,
-            attention_weights,
-        )
-        return output
-
-    @staticmethod
-    @once_differentiable
-    def backward(ctx, grad_output):
-        (
-            value,
-            value_spatial_shapes,
-            value_level_start_index,
-            sampling_locations,
-            attention_weights,
-        ) = ctx.saved_tensors
-        grad_value, grad_sampling_loc, grad_attn_weight = _C.ms_deform_attn_backward(
-            value,
-            value_spatial_shapes,
-            value_level_start_index,
-            sampling_locations,
-            attention_weights,
-            grad_output,
-            ctx.im2col_step,
-        )
-
-        return grad_value, None, None, grad_sampling_loc, grad_attn_weight, None
-
-
-def multi_scale_deformable_attn_pytorch(
-    value: torch.Tensor,
-    value_spatial_shapes: torch.Tensor,
-    sampling_locations: torch.Tensor,
-    attention_weights: torch.Tensor,
-) -> torch.Tensor:
-
-    bs, _, num_heads, embed_dims = value.shape
-    _, num_queries, num_heads, num_levels, num_points, _ = sampling_locations.shape
-    value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1)
-    sampling_grids = 2 * sampling_locations - 1
-    sampling_value_list = []
-    for level, (H_, W_) in enumerate(value_spatial_shapes):
-        # bs, H_*W_, num_heads, embed_dims ->
-        # bs, H_*W_, num_heads*embed_dims ->
-        # bs, num_heads*embed_dims, H_*W_ ->
-        # bs*num_heads, embed_dims, H_, W_
-        value_l_ = (
-            value_list[level].flatten(2).transpose(1, 2).reshape(bs * num_heads, embed_dims, H_, W_)
-        )
-        # bs, num_queries, num_heads, num_points, 2 ->
-        # bs, num_heads, num_queries, num_points, 2 ->
-        # bs*num_heads, num_queries, num_points, 2
-        sampling_grid_l_ = sampling_grids[:, :, :, level].transpose(1, 2).flatten(0, 1)
-        # bs*num_heads, embed_dims, num_queries, num_points
-        sampling_value_l_ = F.grid_sample(
-            value_l_, sampling_grid_l_, mode="bilinear", padding_mode="zeros", align_corners=False
-        )
-        sampling_value_list.append(sampling_value_l_)
-    # (bs, num_queries, num_heads, num_levels, num_points) ->
-    # (bs, num_heads, num_queries, num_levels, num_points) ->
-    # (bs, num_heads, 1, num_queries, num_levels*num_points)
-    attention_weights = attention_weights.transpose(1, 2).reshape(
-        bs * num_heads, 1, num_queries, num_levels * num_points
-    )
-    output = (
-        (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights)
-        .sum(-1)
-        .view(bs, num_heads * embed_dims, num_queries)
-    )
-    return output.transpose(1, 2).contiguous()
-
-
-class MultiScaleDeformableAttention(nn.Module):
-    """Multi-Scale Deformable Attention Module used in Deformable-DETR
-
-    `Deformable DETR: Deformable Transformers for End-to-End Object Detection.
-    <https://arxiv.org/pdf/2010.04159.pdf>`_.
-
-    Args:
-        embed_dim (int): The embedding dimension of Attention. Default: 256.
-        num_heads (int): The number of attention heads. Default: 8.
-        num_levels (int): The number of feature map used in Attention. Default: 4.
-        num_points (int): The number of sampling points for each query
-            in each head. Default: 4.
-        img2col_steps (int): The step used in image_to_column. Defualt: 64.
-            dropout (float): Dropout layer used in output. Default: 0.1.
-        batch_first (bool): if ``True``, then the input and output tensor will be
-            provided as `(bs, n, embed_dim)`. Default: False. `(n, bs, embed_dim)`
-    """
-
-    def __init__(
-        self,
-        embed_dim: int = 256,
-        num_heads: int = 8,
-        num_levels: int = 4,
-        num_points: int = 4,
-        img2col_step: int = 64,
-        batch_first: bool = False,
-    ):
-        super().__init__()
-        if embed_dim % num_heads != 0:
-            raise ValueError(
-                "embed_dim must be divisible by num_heads, but got {} and {}".format(
-                    embed_dim, num_heads
-                )
-            )
-        head_dim = embed_dim // num_heads
-
-        self.batch_first = batch_first
-
-        if not _is_power_of_2(head_dim):
-            warnings.warn(
-                """
-                You'd better set d_model in MSDeformAttn to make sure that
-                each dim of the attention head a power of 2, which is more efficient.
-                """
-            )
-
-        self.im2col_step = img2col_step
-        self.embed_dim = embed_dim
-        self.num_heads = num_heads
-        self.num_levels = num_levels
-        self.num_points = num_points
-        self.sampling_offsets = nn.Linear(embed_dim, num_heads * num_levels * num_points * 2)
-        self.attention_weights = nn.Linear(embed_dim, num_heads * num_levels * num_points)
-        self.value_proj = nn.Linear(embed_dim, embed_dim)
-        self.output_proj = nn.Linear(embed_dim, embed_dim)
-
-        self.init_weights()
-
-    def _reset_parameters(self):
-        return self.init_weights()
-
-    def init_weights(self):
-        """
-        Default initialization for Parameters of Module.
-        """
-        constant_(self.sampling_offsets.weight.data, 0.0)
-        thetas = torch.arange(self.num_heads, dtype=torch.float32) * (
-            2.0 * math.pi / self.num_heads
-        )
-        grid_init = torch.stack([thetas.cos(), thetas.sin()], -1)
-        grid_init = (
-            (grid_init / grid_init.abs().max(-1, keepdim=True)[0])
-            .view(self.num_heads, 1, 1, 2)
-            .repeat(1, self.num_levels, self.num_points, 1)
-        )
-        for i in range(self.num_points):
-            grid_init[:, :, i, :] *= i + 1
-        with torch.no_grad():
-            self.sampling_offsets.bias = nn.Parameter(grid_init.view(-1))
-        constant_(self.attention_weights.weight.data, 0.0)
-        constant_(self.attention_weights.bias.data, 0.0)
-        xavier_uniform_(self.value_proj.weight.data)
-        constant_(self.value_proj.bias.data, 0.0)
-        xavier_uniform_(self.output_proj.weight.data)
-        constant_(self.output_proj.bias.data, 0.0)
-
-    def freeze_sampling_offsets(self):
-        print("Freeze sampling offsets")
-        self.sampling_offsets.weight.requires_grad = False
-        self.sampling_offsets.bias.requires_grad = False
-
-    def freeze_attention_weights(self):
-        print("Freeze attention weights")
-        self.attention_weights.weight.requires_grad = False
-        self.attention_weights.bias.requires_grad = False
-
-    def forward(
-        self,
-        query: torch.Tensor,
-        key: Optional[torch.Tensor] = None,
-        value: Optional[torch.Tensor] = None,
-        query_pos: Optional[torch.Tensor] = None,
-        key_padding_mask: Optional[torch.Tensor] = None,
-        reference_points: Optional[torch.Tensor] = None,
-        spatial_shapes: Optional[torch.Tensor] = None,
-        level_start_index: Optional[torch.Tensor] = None,
-        **kwargs
-    ) -> torch.Tensor:
-
-        """Forward Function of MultiScaleDeformableAttention
-
-        Args:
-            query (torch.Tensor): Query embeddings with shape
-                `(num_query, bs, embed_dim)`
-            key (torch.Tensor): Key embeddings with shape
-                `(num_key, bs, embed_dim)`
-            value (torch.Tensor): Value embeddings with shape
-                `(num_key, bs, embed_dim)`
-            query_pos (torch.Tensor): The position embedding for `query`. Default: None.
-            key_padding_mask (torch.Tensor): ByteTensor for `query`, with shape `(bs, num_key)`,
-                indicating which elements within `key` to be ignored in attention.
-            reference_points (torch.Tensor): The normalized reference points
-                with shape `(bs, num_query, num_levels, 2)`,
-                all elements is range in [0, 1], top-left (0, 0),
-                bottom-right (1, 1), including padding are.
-                or `(N, Length_{query}, num_levels, 4)`, add additional
-                two dimensions `(h, w)` to form reference boxes.
-            spatial_shapes (torch.Tensor): Spatial shape of features in different levels.
-                With shape `(num_levels, 2)`, last dimension represents `(h, w)`.
-            level_start_index (torch.Tensor): The start index of each level. A tensor with
-                shape `(num_levels, )` which can be represented as
-                `[0, h_0 * w_0, h_0 * w_0 + h_1 * w_1, ...]`.
-
-        Returns:
-            torch.Tensor: forward results with shape `(num_query, bs, embed_dim)`
-        """
-
-        if value is None:
-            value = query
-
-        if query_pos is not None:
-            query = query + query_pos
-
-        if not self.batch_first:
-            # change to (bs, num_query ,embed_dims)
-            query = query.permute(1, 0, 2)
-            value = value.permute(1, 0, 2)
-
-        bs, num_query, _ = query.shape
-        bs, num_value, _ = value.shape
-
-        assert (spatial_shapes[:, 0] * spatial_shapes[:, 1]).sum() == num_value
-
-        value = self.value_proj(value)
-        if key_padding_mask is not None:
-            value = value.masked_fill(key_padding_mask[..., None], float(0))
-        value = value.view(bs, num_value, self.num_heads, -1)
-        sampling_offsets = self.sampling_offsets(query).view(
-            bs, num_query, self.num_heads, self.num_levels, self.num_points, 2
-        )
-        attention_weights = self.attention_weights(query).view(
-            bs, num_query, self.num_heads, self.num_levels * self.num_points
-        )
-        attention_weights = attention_weights.softmax(-1)
-        attention_weights = attention_weights.view(
-            bs,
-            num_query,
-            self.num_heads,
-            self.num_levels,
-            self.num_points,
-        )
-
-        # bs, num_query, num_heads, num_levels, num_points, 2
-        if reference_points.shape[-1] == 2:
-            offset_normalizer = torch.stack([spatial_shapes[..., 1], spatial_shapes[..., 0]], -1)
-            sampling_locations = (
-                reference_points[:, :, None, :, None, :]
-                + sampling_offsets / offset_normalizer[None, None, None, :, None, :]
-            )
-        elif reference_points.shape[-1] == 4:
-            sampling_locations = (
-                reference_points[:, :, None, :, None, :2]
-                + sampling_offsets
-                / self.num_points
-                * reference_points[:, :, None, :, None, 2:]
-                * 0.5
-            )
-        else:
-            raise ValueError(
-                "Last dim of reference_points must be 2 or 4, but get {} instead.".format(
-                    reference_points.shape[-1]
-                )
-            )
-    
-        if torch.cuda.is_available() and value.is_cuda:
-            halffloat = False
-            if value.dtype == torch.float16:
-                halffloat = True
-                value = value.float()
-                sampling_locations = sampling_locations.float()
-                attention_weights = attention_weights.float()
-
-            output = MultiScaleDeformableAttnFunction.apply(
-                value,
-                spatial_shapes,
-                level_start_index,
-                sampling_locations,
-                attention_weights,
-                self.im2col_step,
-            )
-
-            if halffloat:
-                output = output.half()
-        else:
-            output = multi_scale_deformable_attn_pytorch(
-                value, spatial_shapes, sampling_locations, attention_weights
-            )
-
-        output = self.output_proj(output)
-
-        if not self.batch_first:
-            output = output.permute(1, 0, 2)
-
-        return output
-
-
-def create_dummy_class(klass, dependency, message=""):
-    """
-    When a dependency of a class is not available, create a dummy class which throws ImportError
-    when used.
-
-    Args:
-        klass (str): name of the class.
-        dependency (str): name of the dependency.
-        message: extra message to print
-    Returns:
-        class: a class object
-    """
-    err = "Cannot import '{}', therefore '{}' is not available.".format(dependency, klass)
-    if message:
-        err = err + " " + message
-
-    class _DummyMetaClass(type):
-        # throw error on class attribute access
-        def __getattr__(_, __):  # noqa: B902
-            raise ImportError(err)
-
-    class _Dummy(object, metaclass=_DummyMetaClass):
-        # throw error on constructor
-        def __init__(self, *args, **kwargs):
-            raise ImportError(err)
-
-    return _Dummy
-
-
-def create_dummy_func(func, dependency, message=""):
-    """
-    When a dependency of a function is not available, create a dummy function which throws
-    ImportError when used.
-
-    Args:
-        func (str): name of the function.
-        dependency (str or list[str]): name(s) of the dependency.
-        message: extra message to print
-    Returns:
-        function: a function object
-    """
-    err = "Cannot import '{}', therefore '{}' is not available.".format(dependency, func)
-    if message:
-        err = err + " " + message
-
-    if isinstance(dependency, (list, tuple)):
-        dependency = ",".join(dependency)
-
-    def _dummy(*args, **kwargs):
-        raise ImportError(err)
-
-    return _dummy

grounding_dino/groundingdino/models/GroundingDINO/transformer.py

@@ -1,959 +0,0 @@
-# ------------------------------------------------------------------------
-# Grounding DINO
-# url: https://github.com/IDEA-Research/GroundingDINO
-# Copyright (c) 2023 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# DINO
-# Copyright (c) 2022 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# Conditional DETR Transformer class.
-# Copyright (c) 2021 Microsoft. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# Modified from DETR (https://github.com/facebookresearch/detr)
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-# ------------------------------------------------------------------------
-
-from typing import Optional
-
-import torch
-import torch.utils.checkpoint as checkpoint
-from torch import Tensor, nn
-
-from grounding_dino.groundingdino.util.misc import inverse_sigmoid
-
-from .fuse_modules import BiAttentionBlock
-from .ms_deform_attn import MultiScaleDeformableAttention as MSDeformAttn
-from .transformer_vanilla import TransformerEncoderLayer
-from .utils import (
-    MLP,
-    _get_activation_fn,
-    _get_clones,
-    gen_encoder_output_proposals,
-    gen_sineembed_for_position,
-    get_sine_pos_embed,
-)
-
-
-class Transformer(nn.Module):
-    def __init__(
-        self,
-        d_model=256,
-        nhead=8,
-        num_queries=300,
-        num_encoder_layers=6,
-        num_unicoder_layers=0,
-        num_decoder_layers=6,
-        dim_feedforward=2048,
-        dropout=0.0,
-        activation="relu",
-        normalize_before=False,
-        return_intermediate_dec=False,
-        query_dim=4,
-        num_patterns=0,
-        # for deformable encoder
-        num_feature_levels=1,
-        enc_n_points=4,
-        dec_n_points=4,
-        # init query
-        learnable_tgt_init=False,
-        # two stage
-        two_stage_type="no",  # ['no', 'standard', 'early', 'combine', 'enceachlayer', 'enclayer1']
-        embed_init_tgt=False,
-        # for text
-        use_text_enhancer=False,
-        use_fusion_layer=False,
-        use_checkpoint=False,
-        use_transformer_ckpt=False,
-        use_text_cross_attention=False,
-        text_dropout=0.1,
-        fusion_dropout=0.1,
-        fusion_droppath=0.0,
-    ):
-        super().__init__()
-        self.num_feature_levels = num_feature_levels
-        self.num_encoder_layers = num_encoder_layers
-        self.num_unicoder_layers = num_unicoder_layers
-        self.num_decoder_layers = num_decoder_layers
-        self.num_queries = num_queries
-        assert query_dim == 4
-
-        # choose encoder layer type
-        encoder_layer = DeformableTransformerEncoderLayer(
-            d_model, dim_feedforward, dropout, activation, num_feature_levels, nhead, enc_n_points
-        )
-
-        if use_text_enhancer:
-            text_enhance_layer = TransformerEncoderLayer(
-                d_model=d_model,
-                nhead=nhead // 2,
-                dim_feedforward=dim_feedforward // 2,
-                dropout=text_dropout,
-            )
-        else:
-            text_enhance_layer = None
-
-        if use_fusion_layer:
-            feature_fusion_layer = BiAttentionBlock(
-                v_dim=d_model,
-                l_dim=d_model,
-                embed_dim=dim_feedforward // 2,
-                num_heads=nhead // 2,
-                dropout=fusion_dropout,
-                drop_path=fusion_droppath,
-            )
-        else:
-            feature_fusion_layer = None
-
-        encoder_norm = nn.LayerNorm(d_model) if normalize_before else None
-        assert encoder_norm is None
-        self.encoder = TransformerEncoder(
-            encoder_layer,
-            num_encoder_layers,
-            d_model=d_model,
-            num_queries=num_queries,
-            text_enhance_layer=text_enhance_layer,
-            feature_fusion_layer=feature_fusion_layer,
-            use_checkpoint=use_checkpoint,
-            use_transformer_ckpt=use_transformer_ckpt,
-        )
-
-        # choose decoder layer type
-        decoder_layer = DeformableTransformerDecoderLayer(
-            d_model,
-            dim_feedforward,
-            dropout,
-            activation,
-            num_feature_levels,
-            nhead,
-            dec_n_points,
-            use_text_cross_attention=use_text_cross_attention,
-        )
-
-        decoder_norm = nn.LayerNorm(d_model)
-        self.decoder = TransformerDecoder(
-            decoder_layer,
-            num_decoder_layers,
-            decoder_norm,
-            return_intermediate=return_intermediate_dec,
-            d_model=d_model,
-            query_dim=query_dim,
-            num_feature_levels=num_feature_levels,
-        )
-
-        self.d_model = d_model
-        self.nhead = nhead
-        self.dec_layers = num_decoder_layers
-        self.num_queries = num_queries  # useful for single stage model only
-        self.num_patterns = num_patterns
-        if not isinstance(num_patterns, int):
-            Warning("num_patterns should be int but {}".format(type(num_patterns)))
-            self.num_patterns = 0
-
-        if num_feature_levels > 1:
-            if self.num_encoder_layers > 0:
-                self.level_embed = nn.Parameter(torch.Tensor(num_feature_levels, d_model))
-            else:
-                self.level_embed = None
-
-        self.learnable_tgt_init = learnable_tgt_init
-        assert learnable_tgt_init, "why not learnable_tgt_init"
-        self.embed_init_tgt = embed_init_tgt
-        if (two_stage_type != "no" and embed_init_tgt) or (two_stage_type == "no"):
-            self.tgt_embed = nn.Embedding(self.num_queries, d_model)
-            nn.init.normal_(self.tgt_embed.weight.data)
-        else:
-            self.tgt_embed = None
-
-        # for two stage
-        self.two_stage_type = two_stage_type
-        assert two_stage_type in ["no", "standard"], "unknown param {} of two_stage_type".format(
-            two_stage_type
-        )
-        if two_stage_type == "standard":
-            # anchor selection at the output of encoder
-            self.enc_output = nn.Linear(d_model, d_model)
-            self.enc_output_norm = nn.LayerNorm(d_model)
-            self.two_stage_wh_embedding = None
-
-        if two_stage_type == "no":
-            self.init_ref_points(num_queries)  # init self.refpoint_embed
-
-        self.enc_out_class_embed = None
-        self.enc_out_bbox_embed = None
-
-        self._reset_parameters()
-
-    def _reset_parameters(self):
-        for p in self.parameters():
-            if p.dim() > 1:
-                nn.init.xavier_uniform_(p)
-        for m in self.modules():
-            if isinstance(m, MSDeformAttn):
-                m._reset_parameters()
-        if self.num_feature_levels > 1 and self.level_embed is not None:
-            nn.init.normal_(self.level_embed)
-
-    def get_valid_ratio(self, mask):
-        _, H, W = mask.shape
-        valid_H = torch.sum(~mask[:, :, 0], 1)
-        valid_W = torch.sum(~mask[:, 0, :], 1)
-        valid_ratio_h = valid_H.float() / H
-        valid_ratio_w = valid_W.float() / W
-        valid_ratio = torch.stack([valid_ratio_w, valid_ratio_h], -1)
-        return valid_ratio
-
-    def init_ref_points(self, use_num_queries):
-        self.refpoint_embed = nn.Embedding(use_num_queries, 4)
-
-    def forward(self, srcs, masks, refpoint_embed, pos_embeds, tgt, attn_mask=None, text_dict=None):
-        """
-        Input:
-            - srcs: List of multi features [bs, ci, hi, wi]
-            - masks: List of multi masks [bs, hi, wi]
-            - refpoint_embed: [bs, num_dn, 4]. None in infer
-            - pos_embeds: List of multi pos embeds [bs, ci, hi, wi]
-            - tgt: [bs, num_dn, d_model]. None in infer
-
-        """
-        # prepare input for encoder
-        src_flatten = []
-        mask_flatten = []
-        lvl_pos_embed_flatten = []
-        spatial_shapes = []
-        for lvl, (src, mask, pos_embed) in enumerate(zip(srcs, masks, pos_embeds)):
-            bs, c, h, w = src.shape
-            spatial_shape = (h, w)
-            spatial_shapes.append(spatial_shape)
-
-            src = src.flatten(2).transpose(1, 2)  # bs, hw, c
-            mask = mask.flatten(1)  # bs, hw
-            pos_embed = pos_embed.flatten(2).transpose(1, 2)  # bs, hw, c
-            if self.num_feature_levels > 1 and self.level_embed is not None:
-                lvl_pos_embed = pos_embed + self.level_embed[lvl].view(1, 1, -1)
-            else:
-                lvl_pos_embed = pos_embed
-            lvl_pos_embed_flatten.append(lvl_pos_embed)
-            src_flatten.append(src)
-            mask_flatten.append(mask)
-        src_flatten = torch.cat(src_flatten, 1)  # bs, \sum{hxw}, c
-        mask_flatten = torch.cat(mask_flatten, 1)  # bs, \sum{hxw}
-        lvl_pos_embed_flatten = torch.cat(lvl_pos_embed_flatten, 1)  # bs, \sum{hxw}, c
-        spatial_shapes = torch.as_tensor(
-            spatial_shapes, dtype=torch.long, device=src_flatten.device
-        )
-        level_start_index = torch.cat(
-            (spatial_shapes.new_zeros((1,)), spatial_shapes.prod(1).cumsum(0)[:-1])
-        )
-        valid_ratios = torch.stack([self.get_valid_ratio(m) for m in masks], 1)
-
-        # two stage
-        enc_topk_proposals = enc_refpoint_embed = None
-
-        #########################################################
-        # Begin Encoder
-        #########################################################
-        memory, memory_text = self.encoder(
-            src_flatten,
-            pos=lvl_pos_embed_flatten,
-            level_start_index=level_start_index,
-            spatial_shapes=spatial_shapes,
-            valid_ratios=valid_ratios,
-            key_padding_mask=mask_flatten,
-            memory_text=text_dict["encoded_text"],
-            text_attention_mask=~text_dict["text_token_mask"],
-            # we ~ the mask . False means use the token; True means pad the token
-            position_ids=text_dict["position_ids"],
-            text_self_attention_masks=text_dict["text_self_attention_masks"],
-        )
-        #########################################################
-        # End Encoder
-        # - memory: bs, \sum{hw}, c
-        # - mask_flatten: bs, \sum{hw}
-        # - lvl_pos_embed_flatten: bs, \sum{hw}, c
-        # - enc_intermediate_output: None or (nenc+1, bs, nq, c) or (nenc, bs, nq, c)
-        # - enc_intermediate_refpoints: None or (nenc+1, bs, nq, c) or (nenc, bs, nq, c)
-        #########################################################
-        text_dict["encoded_text"] = memory_text
-        # if os.environ.get("SHILONG_AMP_INFNAN_DEBUG") == '1':
-        #     if memory.isnan().any() | memory.isinf().any():
-        #         import ipdb; ipdb.set_trace()
-
-        if self.two_stage_type == "standard":
-            output_memory, output_proposals = gen_encoder_output_proposals(
-                memory, mask_flatten, spatial_shapes
-            )
-            output_memory = self.enc_output_norm(self.enc_output(output_memory))
-
-            if text_dict is not None:
-                enc_outputs_class_unselected = self.enc_out_class_embed(output_memory, text_dict)
-            else:
-                enc_outputs_class_unselected = self.enc_out_class_embed(output_memory)
-
-            topk_logits = enc_outputs_class_unselected.max(-1)[0]
-            enc_outputs_coord_unselected = (
-                self.enc_out_bbox_embed(output_memory) + output_proposals
-            )  # (bs, \sum{hw}, 4) unsigmoid
-            topk = self.num_queries
-
-            topk_proposals = torch.topk(topk_logits, topk, dim=1)[1]  # bs, nq
-
-            # gather boxes
-            refpoint_embed_undetach = torch.gather(
-                enc_outputs_coord_unselected, 1, topk_proposals.unsqueeze(-1).repeat(1, 1, 4)
-            )  # unsigmoid
-            refpoint_embed_ = refpoint_embed_undetach.detach()
-            init_box_proposal = torch.gather(
-                output_proposals, 1, topk_proposals.unsqueeze(-1).repeat(1, 1, 4)
-            ).sigmoid()  # sigmoid
-
-            # gather tgt
-            tgt_undetach = torch.gather(
-                output_memory, 1, topk_proposals.unsqueeze(-1).repeat(1, 1, self.d_model)
-            )
-            if self.embed_init_tgt:
-                tgt_ = (
-                    self.tgt_embed.weight[:, None, :].repeat(1, bs, 1).transpose(0, 1)
-                )  # nq, bs, d_model
-            else:
-                tgt_ = tgt_undetach.detach()
-
-            if refpoint_embed is not None:
-                refpoint_embed = torch.cat([refpoint_embed, refpoint_embed_], dim=1)
-                tgt = torch.cat([tgt, tgt_], dim=1)
-            else:
-                refpoint_embed, tgt = refpoint_embed_, tgt_
-
-        elif self.two_stage_type == "no":
-            tgt_ = (
-                self.tgt_embed.weight[:, None, :].repeat(1, bs, 1).transpose(0, 1)
-            )  # nq, bs, d_model
-            refpoint_embed_ = (
-                self.refpoint_embed.weight[:, None, :].repeat(1, bs, 1).transpose(0, 1)
-            )  # nq, bs, 4
-
-            if refpoint_embed is not None:
-                refpoint_embed = torch.cat([refpoint_embed, refpoint_embed_], dim=1)
-                tgt = torch.cat([tgt, tgt_], dim=1)
-            else:
-                refpoint_embed, tgt = refpoint_embed_, tgt_
-
-            if self.num_patterns > 0:
-                tgt_embed = tgt.repeat(1, self.num_patterns, 1)
-                refpoint_embed = refpoint_embed.repeat(1, self.num_patterns, 1)
-                tgt_pat = self.patterns.weight[None, :, :].repeat_interleave(
-                    self.num_queries, 1
-                )  # 1, n_q*n_pat, d_model
-                tgt = tgt_embed + tgt_pat
-
-            init_box_proposal = refpoint_embed_.sigmoid()
-
-        else:
-            raise NotImplementedError("unknown two_stage_type {}".format(self.two_stage_type))
-        #########################################################
-        # End preparing tgt
-        # - tgt: bs, NQ, d_model
-        # - refpoint_embed(unsigmoid): bs, NQ, d_model
-        #########################################################
-
-        #########################################################
-        # Begin Decoder
-        #########################################################
-        hs, references = self.decoder(
-            tgt=tgt.transpose(0, 1),
-            memory=memory.transpose(0, 1),
-            memory_key_padding_mask=mask_flatten,
-            pos=lvl_pos_embed_flatten.transpose(0, 1),
-            refpoints_unsigmoid=refpoint_embed.transpose(0, 1),
-            level_start_index=level_start_index,
-            spatial_shapes=spatial_shapes,
-            valid_ratios=valid_ratios,
-            tgt_mask=attn_mask,
-            memory_text=text_dict["encoded_text"],
-            text_attention_mask=~text_dict["text_token_mask"],
-            # we ~ the mask . False means use the token; True means pad the token
-        )
-        #########################################################
-        # End Decoder
-        # hs: n_dec, bs, nq, d_model
-        # references: n_dec+1, bs, nq, query_dim
-        #########################################################
-
-        #########################################################
-        # Begin postprocess
-        #########################################################
-        if self.two_stage_type == "standard":
-            hs_enc = tgt_undetach.unsqueeze(0)
-            ref_enc = refpoint_embed_undetach.sigmoid().unsqueeze(0)
-        else:
-            hs_enc = ref_enc = None
-        #########################################################
-        # End postprocess
-        # hs_enc: (n_enc+1, bs, nq, d_model) or (1, bs, nq, d_model) or (n_enc, bs, nq, d_model) or None
-        # ref_enc: (n_enc+1, bs, nq, query_dim) or (1, bs, nq, query_dim) or (n_enc, bs, nq, d_model) or None
-        #########################################################
-
-        return hs, references, hs_enc, ref_enc, init_box_proposal
-        # hs: (n_dec, bs, nq, d_model)
-        # references: sigmoid coordinates. (n_dec+1, bs, bq, 4)
-        # hs_enc: (n_enc+1, bs, nq, d_model) or (1, bs, nq, d_model) or None
-        # ref_enc: sigmoid coordinates. \
-        #           (n_enc+1, bs, nq, query_dim) or (1, bs, nq, query_dim) or None
-
-
-class TransformerEncoder(nn.Module):
-    def __init__(
-        self,
-        encoder_layer,
-        num_layers,
-        d_model=256,
-        num_queries=300,
-        enc_layer_share=False,
-        text_enhance_layer=None,
-        feature_fusion_layer=None,
-        use_checkpoint=False,
-        use_transformer_ckpt=False,
-    ):
-        """_summary_
-
-        Args:
-            encoder_layer (_type_): _description_
-            num_layers (_type_): _description_
-            norm (_type_, optional): _description_. Defaults to None.
-            d_model (int, optional): _description_. Defaults to 256.
-            num_queries (int, optional): _description_. Defaults to 300.
-            enc_layer_share (bool, optional): _description_. Defaults to False.
-
-        """
-        super().__init__()
-        # prepare layers
-        self.layers = []
-        self.text_layers = []
-        self.fusion_layers = []
-        if num_layers > 0:
-            self.layers = _get_clones(encoder_layer, num_layers, layer_share=enc_layer_share)
-
-            if text_enhance_layer is not None:
-                self.text_layers = _get_clones(
-                    text_enhance_layer, num_layers, layer_share=enc_layer_share
-                )
-            if feature_fusion_layer is not None:
-                self.fusion_layers = _get_clones(
-                    feature_fusion_layer, num_layers, layer_share=enc_layer_share
-                )
-        else:
-            self.layers = []
-            del encoder_layer
-
-            if text_enhance_layer is not None:
-                self.text_layers = []
-                del text_enhance_layer
-            if feature_fusion_layer is not None:
-                self.fusion_layers = []
-                del feature_fusion_layer
-
-        self.query_scale = None
-        self.num_queries = num_queries
-        self.num_layers = num_layers
-        self.d_model = d_model
-
-        self.use_checkpoint = use_checkpoint
-        self.use_transformer_ckpt = use_transformer_ckpt
-
-    @staticmethod
-    def get_reference_points(spatial_shapes, valid_ratios, device):
-        reference_points_list = []
-        for lvl, (H_, W_) in enumerate(spatial_shapes):
-
-            ref_y, ref_x = torch.meshgrid(
-                torch.linspace(0.5, H_ - 0.5, H_, dtype=torch.float32, device=device),
-                torch.linspace(0.5, W_ - 0.5, W_, dtype=torch.float32, device=device),
-            )
-            ref_y = ref_y.reshape(-1)[None] / (valid_ratios[:, None, lvl, 1] * H_)
-            ref_x = ref_x.reshape(-1)[None] / (valid_ratios[:, None, lvl, 0] * W_)
-            ref = torch.stack((ref_x, ref_y), -1)
-            reference_points_list.append(ref)
-        reference_points = torch.cat(reference_points_list, 1)
-        reference_points = reference_points[:, :, None] * valid_ratios[:, None]
-        return reference_points
-
-    def forward(
-        self,
-        # for images
-        src: Tensor,
-        pos: Tensor,
-        spatial_shapes: Tensor,
-        level_start_index: Tensor,
-        valid_ratios: Tensor,
-        key_padding_mask: Tensor,
-        # for texts
-        memory_text: Tensor = None,
-        text_attention_mask: Tensor = None,
-        pos_text: Tensor = None,
-        text_self_attention_masks: Tensor = None,
-        position_ids: Tensor = None,
-    ):
-        """
-        Input:
-            - src: [bs, sum(hi*wi), 256]
-            - pos: pos embed for src. [bs, sum(hi*wi), 256]
-            - spatial_shapes: h,w of each level [num_level, 2]
-            - level_start_index: [num_level] start point of level in sum(hi*wi).
-            - valid_ratios: [bs, num_level, 2]
-            - key_padding_mask: [bs, sum(hi*wi)]
-
-            - memory_text: bs, n_text, 256
-            - text_attention_mask: bs, n_text
-                False for no padding; True for padding
-            - pos_text: bs, n_text, 256
-
-            - position_ids: bs, n_text
-        Intermedia:
-            - reference_points: [bs, sum(hi*wi), num_level, 2]
-        Outpus:
-            - output: [bs, sum(hi*wi), 256]
-        """
-
-        output = src
-
-        # preparation and reshape
-        if self.num_layers > 0:
-            reference_points = self.get_reference_points(
-                spatial_shapes, valid_ratios, device=src.device
-            )
-
-        if self.text_layers:
-            # generate pos_text
-            bs, n_text, text_dim = memory_text.shape
-            if pos_text is None and position_ids is None:
-                pos_text = (
-                    torch.arange(n_text, device=memory_text.device)
-                    .float()
-                    .unsqueeze(0)
-                    .unsqueeze(-1)
-                    .repeat(bs, 1, 1)
-                )
-                pos_text = get_sine_pos_embed(pos_text, num_pos_feats=256, exchange_xy=False)
-            if position_ids is not None:
-                pos_text = get_sine_pos_embed(
-                    position_ids[..., None], num_pos_feats=256, exchange_xy=False
-                )
-
-        # main process
-        for layer_id, layer in enumerate(self.layers):
-            # if output.isnan().any() or memory_text.isnan().any():
-            #     if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO':
-            #         import ipdb; ipdb.set_trace()
-            if self.fusion_layers:
-                if self.use_checkpoint:
-                    output, memory_text = checkpoint.checkpoint(
-                        self.fusion_layers[layer_id],
-                        output,
-                        memory_text,
-                        key_padding_mask,
-                        text_attention_mask,
-                    )
-                else:
-                    output, memory_text = self.fusion_layers[layer_id](
-                        v=output,
-                        l=memory_text,
-                        attention_mask_v=key_padding_mask,
-                        attention_mask_l=text_attention_mask,
-                    )
-
-            if self.text_layers:
-                memory_text = self.text_layers[layer_id](
-                    src=memory_text.transpose(0, 1),
-                    src_mask=~text_self_attention_masks,  # note we use ~ for mask here
-                    src_key_padding_mask=text_attention_mask,
-                    pos=(pos_text.transpose(0, 1) if pos_text is not None else None),
-                ).transpose(0, 1)
-
-            # main process
-            if self.use_transformer_ckpt:
-                output = checkpoint.checkpoint(
-                    layer,
-                    output,
-                    pos,
-                    reference_points,
-                    spatial_shapes,
-                    level_start_index,
-                    key_padding_mask,
-                )
-            else:
-                output = layer(
-                    src=output,
-                    pos=pos,
-                    reference_points=reference_points,
-                    spatial_shapes=spatial_shapes,
-                    level_start_index=level_start_index,
-                    key_padding_mask=key_padding_mask,
-                )
-
-        return output, memory_text
-
-
-class TransformerDecoder(nn.Module):
-    def __init__(
-        self,
-        decoder_layer,
-        num_layers,
-        norm=None,
-        return_intermediate=False,
-        d_model=256,
-        query_dim=4,
-        num_feature_levels=1,
-    ):
-        super().__init__()
-        if num_layers > 0:
-            self.layers = _get_clones(decoder_layer, num_layers)
-        else:
-            self.layers = []
-        self.num_layers = num_layers
-        self.norm = norm
-        self.return_intermediate = return_intermediate
-        assert return_intermediate, "support return_intermediate only"
-        self.query_dim = query_dim
-        assert query_dim in [2, 4], "query_dim should be 2/4 but {}".format(query_dim)
-        self.num_feature_levels = num_feature_levels
-
-        self.ref_point_head = MLP(query_dim // 2 * d_model, d_model, d_model, 2)
-        self.query_pos_sine_scale = None
-
-        self.query_scale = None
-        self.bbox_embed = None
-        self.class_embed = None
-
-        self.d_model = d_model
-
-        self.ref_anchor_head = None
-
-    def forward(
-        self,
-        tgt,
-        memory,
-        tgt_mask: Optional[Tensor] = None,
-        memory_mask: Optional[Tensor] = None,
-        tgt_key_padding_mask: Optional[Tensor] = None,
-        memory_key_padding_mask: Optional[Tensor] = None,
-        pos: Optional[Tensor] = None,
-        refpoints_unsigmoid: Optional[Tensor] = None,  # num_queries, bs, 2
-        # for memory
-        level_start_index: Optional[Tensor] = None,  # num_levels
-        spatial_shapes: Optional[Tensor] = None,  # bs, num_levels, 2
-        valid_ratios: Optional[Tensor] = None,
-        # for text
-        memory_text: Optional[Tensor] = None,
-        text_attention_mask: Optional[Tensor] = None,
-    ):
-        """
-        Input:
-            - tgt: nq, bs, d_model
-            - memory: hw, bs, d_model
-            - pos: hw, bs, d_model
-            - refpoints_unsigmoid: nq, bs, 2/4
-            - valid_ratios/spatial_shapes: bs, nlevel, 2
-        """
-        output = tgt
-
-        intermediate = []
-        reference_points = refpoints_unsigmoid.sigmoid()
-        ref_points = [reference_points]
-
-        for layer_id, layer in enumerate(self.layers):
-
-            if reference_points.shape[-1] == 4:
-                reference_points_input = (
-                    reference_points[:, :, None]
-                    * torch.cat([valid_ratios, valid_ratios], -1)[None, :]
-                )  # nq, bs, nlevel, 4
-            else:
-                assert reference_points.shape[-1] == 2
-                reference_points_input = reference_points[:, :, None] * valid_ratios[None, :]
-            query_sine_embed = gen_sineembed_for_position(
-                reference_points_input[:, :, 0, :]
-            )  # nq, bs, 256*2
-
-            # conditional query
-            raw_query_pos = self.ref_point_head(query_sine_embed)  # nq, bs, 256
-            pos_scale = self.query_scale(output) if self.query_scale is not None else 1
-            query_pos = pos_scale * raw_query_pos
-            # if os.environ.get("SHILONG_AMP_INFNAN_DEBUG") == '1':
-            #     if query_pos.isnan().any() | query_pos.isinf().any():
-            #         import ipdb; ipdb.set_trace()
-
-            # main process
-            output = layer(
-                tgt=output,
-                tgt_query_pos=query_pos,
-                tgt_query_sine_embed=query_sine_embed,
-                tgt_key_padding_mask=tgt_key_padding_mask,
-                tgt_reference_points=reference_points_input,
-                memory_text=memory_text,
-                text_attention_mask=text_attention_mask,
-                memory=memory,
-                memory_key_padding_mask=memory_key_padding_mask,
-                memory_level_start_index=level_start_index,
-                memory_spatial_shapes=spatial_shapes,
-                memory_pos=pos,
-                self_attn_mask=tgt_mask,
-                cross_attn_mask=memory_mask,
-            )
-            if output.isnan().any() | output.isinf().any():
-                print(f"output layer_id {layer_id} is nan")
-                try:
-                    num_nan = output.isnan().sum().item()
-                    num_inf = output.isinf().sum().item()
-                    print(f"num_nan {num_nan}, num_inf {num_inf}")
-                except Exception as e:
-                    print(e)
-                    # if os.environ.get("SHILONG_AMP_INFNAN_DEBUG") == '1':
-                    #     import ipdb; ipdb.set_trace()
-
-            # iter update
-            if self.bbox_embed is not None:
-                # box_holder = self.bbox_embed(output)
-                # box_holder[..., :self.query_dim] += inverse_sigmoid(reference_points)
-                # new_reference_points = box_holder[..., :self.query_dim].sigmoid()
-
-                reference_before_sigmoid = inverse_sigmoid(reference_points)
-                delta_unsig = self.bbox_embed[layer_id](output)
-                outputs_unsig = delta_unsig + reference_before_sigmoid
-                new_reference_points = outputs_unsig.sigmoid()
-
-                reference_points = new_reference_points.detach()
-                # if layer_id != self.num_layers - 1:
-                ref_points.append(new_reference_points)
-
-            intermediate.append(self.norm(output))
-
-        return [
-            [itm_out.transpose(0, 1) for itm_out in intermediate],
-            [itm_refpoint.transpose(0, 1) for itm_refpoint in ref_points],
-        ]
-
-
-class DeformableTransformerEncoderLayer(nn.Module):
-    def __init__(
-        self,
-        d_model=256,
-        d_ffn=1024,
-        dropout=0.1,
-        activation="relu",
-        n_levels=4,
-        n_heads=8,
-        n_points=4,
-    ):
-        super().__init__()
-
-        # self attention
-        self.self_attn = MSDeformAttn(
-            embed_dim=d_model,
-            num_levels=n_levels,
-            num_heads=n_heads,
-            num_points=n_points,
-            batch_first=True,
-        )
-        self.dropout1 = nn.Dropout(dropout)
-        self.norm1 = nn.LayerNorm(d_model)
-
-        # ffn
-        self.linear1 = nn.Linear(d_model, d_ffn)
-        self.activation = _get_activation_fn(activation, d_model=d_ffn)
-        self.dropout2 = nn.Dropout(dropout)
-        self.linear2 = nn.Linear(d_ffn, d_model)
-        self.dropout3 = nn.Dropout(dropout)
-        self.norm2 = nn.LayerNorm(d_model)
-
-    @staticmethod
-    def with_pos_embed(tensor, pos):
-        return tensor if pos is None else tensor + pos
-
-    def forward_ffn(self, src):
-        src2 = self.linear2(self.dropout2(self.activation(self.linear1(src))))
-        src = src + self.dropout3(src2)
-        src = self.norm2(src)
-        return src
-
-    def forward(
-        self, src, pos, reference_points, spatial_shapes, level_start_index, key_padding_mask=None
-    ):
-        # self attention
-        # import ipdb; ipdb.set_trace()
-        src2 = self.self_attn(
-            query=self.with_pos_embed(src, pos),
-            reference_points=reference_points,
-            value=src,
-            spatial_shapes=spatial_shapes,
-            level_start_index=level_start_index,
-            key_padding_mask=key_padding_mask,
-        )
-        src = src + self.dropout1(src2)
-        src = self.norm1(src)
-
-        # ffn
-        src = self.forward_ffn(src)
-
-        return src
-
-
-class DeformableTransformerDecoderLayer(nn.Module):
-    def __init__(
-        self,
-        d_model=256,
-        d_ffn=1024,
-        dropout=0.1,
-        activation="relu",
-        n_levels=4,
-        n_heads=8,
-        n_points=4,
-        use_text_feat_guide=False,
-        use_text_cross_attention=False,
-    ):
-        super().__init__()
-
-        # cross attention
-        self.cross_attn = MSDeformAttn(
-            embed_dim=d_model,
-            num_levels=n_levels,
-            num_heads=n_heads,
-            num_points=n_points,
-            batch_first=True,
-        )
-        self.dropout1 = nn.Dropout(dropout) if dropout > 0 else nn.Identity()
-        self.norm1 = nn.LayerNorm(d_model)
-
-        # cross attention text
-        if use_text_cross_attention:
-            self.ca_text = nn.MultiheadAttention(d_model, n_heads, dropout=dropout)
-            self.catext_dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity()
-            self.catext_norm = nn.LayerNorm(d_model)
-
-        # self attention
-        self.self_attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout)
-        self.dropout2 = nn.Dropout(dropout) if dropout > 0 else nn.Identity()
-        self.norm2 = nn.LayerNorm(d_model)
-
-        # ffn
-        self.linear1 = nn.Linear(d_model, d_ffn)
-        self.activation = _get_activation_fn(activation, d_model=d_ffn, batch_dim=1)
-        self.dropout3 = nn.Dropout(dropout) if dropout > 0 else nn.Identity()
-        self.linear2 = nn.Linear(d_ffn, d_model)
-        self.dropout4 = nn.Dropout(dropout) if dropout > 0 else nn.Identity()
-        self.norm3 = nn.LayerNorm(d_model)
-
-        self.key_aware_proj = None
-        self.use_text_feat_guide = use_text_feat_guide
-        assert not use_text_feat_guide
-        self.use_text_cross_attention = use_text_cross_attention
-
-    def rm_self_attn_modules(self):
-        self.self_attn = None
-        self.dropout2 = None
-        self.norm2 = None
-
-    @staticmethod
-    def with_pos_embed(tensor, pos):
-        return tensor if pos is None else tensor + pos
-
-    def forward_ffn(self, tgt):
-        with torch.cuda.amp.autocast(enabled=False):
-            tgt2 = self.linear2(self.dropout3(self.activation(self.linear1(tgt))))
-        tgt = tgt + self.dropout4(tgt2)
-        tgt = self.norm3(tgt)
-        return tgt
-
-    def forward(
-        self,
-        # for tgt
-        tgt: Optional[Tensor],  # nq, bs, d_model
-        tgt_query_pos: Optional[Tensor] = None,  # pos for query. MLP(Sine(pos))
-        tgt_query_sine_embed: Optional[Tensor] = None,  # pos for query. Sine(pos)
-        tgt_key_padding_mask: Optional[Tensor] = None,
-        tgt_reference_points: Optional[Tensor] = None,  # nq, bs, 4
-        memory_text: Optional[Tensor] = None,  # bs, num_token, d_model
-        text_attention_mask: Optional[Tensor] = None,  # bs, num_token
-        # for memory
-        memory: Optional[Tensor] = None,  # hw, bs, d_model
-        memory_key_padding_mask: Optional[Tensor] = None,
-        memory_level_start_index: Optional[Tensor] = None,  # num_levels
-        memory_spatial_shapes: Optional[Tensor] = None,  # bs, num_levels, 2
-        memory_pos: Optional[Tensor] = None,  # pos for memory
-        # sa
-        self_attn_mask: Optional[Tensor] = None,  # mask used for self-attention
-        cross_attn_mask: Optional[Tensor] = None,  # mask used for cross-attention
-    ):
-        """
-        Input:
-            - tgt/tgt_query_pos: nq, bs, d_model
-            -
-        """
-        assert cross_attn_mask is None
-
-        # self attention
-        if self.self_attn is not None:
-            # import ipdb; ipdb.set_trace()
-            q = k = self.with_pos_embed(tgt, tgt_query_pos)
-            tgt2 = self.self_attn(q, k, tgt, attn_mask=self_attn_mask)[0]
-            tgt = tgt + self.dropout2(tgt2)
-            tgt = self.norm2(tgt)
-
-        if self.use_text_cross_attention:
-            tgt2 = self.ca_text(
-                self.with_pos_embed(tgt, tgt_query_pos),
-                memory_text.transpose(0, 1),
-                memory_text.transpose(0, 1),
-                key_padding_mask=text_attention_mask,
-            )[0]
-            tgt = tgt + self.catext_dropout(tgt2)
-            tgt = self.catext_norm(tgt)
-
-        tgt2 = self.cross_attn(
-            query=self.with_pos_embed(tgt, tgt_query_pos).transpose(0, 1),
-            reference_points=tgt_reference_points.transpose(0, 1).contiguous(),
-            value=memory.transpose(0, 1),
-            spatial_shapes=memory_spatial_shapes,
-            level_start_index=memory_level_start_index,
-            key_padding_mask=memory_key_padding_mask,
-        ).transpose(0, 1)
-        tgt = tgt + self.dropout1(tgt2)
-        tgt = self.norm1(tgt)
-
-        # ffn
-        tgt = self.forward_ffn(tgt)
-
-        return tgt
-
-
-def build_transformer(args):
-    return Transformer(
-        d_model=args.hidden_dim,
-        dropout=args.dropout,
-        nhead=args.nheads,
-        num_queries=args.num_queries,
-        dim_feedforward=args.dim_feedforward,
-        num_encoder_layers=args.enc_layers,
-        num_decoder_layers=args.dec_layers,
-        normalize_before=args.pre_norm,
-        return_intermediate_dec=True,
-        query_dim=args.query_dim,
-        activation=args.transformer_activation,
-        num_patterns=args.num_patterns,
-        num_feature_levels=args.num_feature_levels,
-        enc_n_points=args.enc_n_points,
-        dec_n_points=args.dec_n_points,
-        learnable_tgt_init=True,
-        # two stage
-        two_stage_type=args.two_stage_type,  # ['no', 'standard', 'early']
-        embed_init_tgt=args.embed_init_tgt,
-        use_text_enhancer=args.use_text_enhancer,
-        use_fusion_layer=args.use_fusion_layer,
-        use_checkpoint=args.use_checkpoint,
-        use_transformer_ckpt=args.use_transformer_ckpt,
-        use_text_cross_attention=args.use_text_cross_attention,
-        text_dropout=args.text_dropout,
-        fusion_dropout=args.fusion_dropout,
-        fusion_droppath=args.fusion_droppath,
-    )

grounding_dino/groundingdino/models/GroundingDINO/transformer_vanilla.py

@@ -1,123 +0,0 @@
-# ------------------------------------------------------------------------
-# Grounding DINO
-# url: https://github.com/IDEA-Research/GroundingDINO
-# Copyright (c) 2023 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# Copyright (c) Aishwarya Kamath & Nicolas Carion. Licensed under the Apache License 2.0. All Rights Reserved
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-"""
-DETR Transformer class.
-
-Copy-paste from torch.nn.Transformer with modifications:
-    * positional encodings are passed in MHattention
-    * extra LN at the end of encoder is removed
-    * decoder returns a stack of activations from all decoding layers
-"""
-from typing import Optional
-
-import torch
-import torch.nn.functional as F
-from torch import Tensor, nn
-
-from .utils import (
-    MLP,
-    _get_activation_fn,
-    _get_clones,
-    gen_encoder_output_proposals,
-    gen_sineembed_for_position,
-    sigmoid_focal_loss,
-)
-
-
-class TextTransformer(nn.Module):
-    def __init__(self, num_layers, d_model=256, nheads=8, dim_feedforward=2048, dropout=0.1):
-        super().__init__()
-        self.num_layers = num_layers
-        self.d_model = d_model
-        self.nheads = nheads
-        self.dim_feedforward = dim_feedforward
-        self.norm = None
-
-        single_encoder_layer = TransformerEncoderLayer(
-            d_model=d_model, nhead=nheads, dim_feedforward=dim_feedforward, dropout=dropout
-        )
-        self.layers = _get_clones(single_encoder_layer, num_layers)
-
-    def forward(self, memory_text: torch.Tensor, text_attention_mask: torch.Tensor):
-        """
-
-        Args:
-            text_attention_mask: bs, num_token
-            memory_text: bs, num_token, d_model
-
-        Raises:
-            RuntimeError: _description_
-
-        Returns:
-            output: bs, num_token, d_model
-        """
-
-        output = memory_text.transpose(0, 1)
-
-        for layer in self.layers:
-            output = layer(output, src_key_padding_mask=text_attention_mask)
-
-        if self.norm is not None:
-            output = self.norm(output)
-
-        return output.transpose(0, 1)
-
-
-class TransformerEncoderLayer(nn.Module):
-    def __init__(
-        self,
-        d_model,
-        nhead,
-        dim_feedforward=2048,
-        dropout=0.1,
-        activation="relu",
-        normalize_before=False,
-    ):
-        super().__init__()
-        self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
-        # Implementation of Feedforward model
-        self.linear1 = nn.Linear(d_model, dim_feedforward)
-        self.dropout = nn.Dropout(dropout)
-        self.linear2 = nn.Linear(dim_feedforward, d_model)
-
-        self.norm1 = nn.LayerNorm(d_model)
-        self.norm2 = nn.LayerNorm(d_model)
-        self.dropout1 = nn.Dropout(dropout)
-        self.dropout2 = nn.Dropout(dropout)
-
-        self.activation = _get_activation_fn(activation)
-        self.normalize_before = normalize_before
-        self.nhead = nhead
-
-    def with_pos_embed(self, tensor, pos: Optional[Tensor]):
-        return tensor if pos is None else tensor + pos
-
-    def forward(
-        self,
-        src,
-        src_mask: Optional[Tensor] = None,
-        src_key_padding_mask: Optional[Tensor] = None,
-        pos: Optional[Tensor] = None,
-    ):
-        # repeat attn mask
-        if src_mask.dim() == 3 and src_mask.shape[0] == src.shape[1]:
-            # bs, num_q, num_k
-            src_mask = src_mask.repeat(self.nhead, 1, 1)
-
-        q = k = self.with_pos_embed(src, pos)
-
-        src2 = self.self_attn(q, k, value=src, attn_mask=src_mask)[0]
-
-        # src2 = self.self_attn(q, k, value=src, attn_mask=src_mask, key_padding_mask=src_key_padding_mask)[0]
-        src = src + self.dropout1(src2)
-        src = self.norm1(src)
-        src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
-        src = src + self.dropout2(src2)
-        src = self.norm2(src)
-        return src

grounding_dino/groundingdino/models/GroundingDINO/utils.py

@@ -1,268 +0,0 @@
-# ------------------------------------------------------------------------
-# Grounding DINO
-# url: https://github.com/IDEA-Research/GroundingDINO
-# Copyright (c) 2023 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-
-import copy
-import math
-
-import torch
-import torch.nn.functional as F
-from torch import Tensor, nn
-
-
-def _get_clones(module, N, layer_share=False):
-    # import ipdb; ipdb.set_trace()
-    if layer_share:
-        return nn.ModuleList([module for i in range(N)])
-    else:
-        return nn.ModuleList([copy.deepcopy(module) for i in range(N)])
-
-
-def get_sine_pos_embed(
-    pos_tensor: torch.Tensor,
-    num_pos_feats: int = 128,
-    temperature: int = 10000,
-    exchange_xy: bool = True,
-):
-    """generate sine position embedding from a position tensor
-    Args:
-        pos_tensor (torch.Tensor): shape: [..., n].
-        num_pos_feats (int): projected shape for each float in the tensor.
-        temperature (int): temperature in the sine/cosine function.
-        exchange_xy (bool, optional): exchange pos x and pos y. \
-            For example, input tensor is [x,y], the results will be [pos(y), pos(x)]. Defaults to True.
-    Returns:
-        pos_embed (torch.Tensor): shape: [..., n*num_pos_feats].
-    """
-    scale = 2 * math.pi
-    dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=pos_tensor.device)
-    dim_t = temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / num_pos_feats)
-
-    def sine_func(x: torch.Tensor):
-        sin_x = x * scale / dim_t
-        sin_x = torch.stack((sin_x[..., 0::2].sin(), sin_x[..., 1::2].cos()), dim=3).flatten(2)
-        return sin_x
-
-    pos_res = [sine_func(x) for x in pos_tensor.split([1] * pos_tensor.shape[-1], dim=-1)]
-    if exchange_xy:
-        pos_res[0], pos_res[1] = pos_res[1], pos_res[0]
-    pos_res = torch.cat(pos_res, dim=-1)
-    return pos_res
-
-
-def gen_encoder_output_proposals(
-    memory: Tensor, memory_padding_mask: Tensor, spatial_shapes: Tensor, learnedwh=None
-):
-    """
-    Input:
-        - memory: bs, \sum{hw}, d_model
-        - memory_padding_mask: bs, \sum{hw}
-        - spatial_shapes: nlevel, 2
-        - learnedwh: 2
-    Output:
-        - output_memory: bs, \sum{hw}, d_model
-        - output_proposals: bs, \sum{hw}, 4
-    """
-    N_, S_, C_ = memory.shape
-    proposals = []
-    _cur = 0
-    for lvl, (H_, W_) in enumerate(spatial_shapes):
-        mask_flatten_ = memory_padding_mask[:, _cur : (_cur + H_ * W_)].view(N_, H_, W_, 1)
-        valid_H = torch.sum(~mask_flatten_[:, :, 0, 0], 1)
-        valid_W = torch.sum(~mask_flatten_[:, 0, :, 0], 1)
-
-        # import ipdb; ipdb.set_trace()
-
-        grid_y, grid_x = torch.meshgrid(
-            torch.linspace(0, H_ - 1, H_, dtype=torch.float32, device=memory.device),
-            torch.linspace(0, W_ - 1, W_, dtype=torch.float32, device=memory.device),
-        )
-        grid = torch.cat([grid_x.unsqueeze(-1), grid_y.unsqueeze(-1)], -1)  # H_, W_, 2
-
-        scale = torch.cat([valid_W.unsqueeze(-1), valid_H.unsqueeze(-1)], 1).view(N_, 1, 1, 2)
-        grid = (grid.unsqueeze(0).expand(N_, -1, -1, -1) + 0.5) / scale
-
-        if learnedwh is not None:
-            # import ipdb; ipdb.set_trace()
-            wh = torch.ones_like(grid) * learnedwh.sigmoid() * (2.0**lvl)
-        else:
-            wh = torch.ones_like(grid) * 0.05 * (2.0**lvl)
-
-        # scale = torch.cat([W_[None].unsqueeze(-1), H_[None].unsqueeze(-1)], 1).view(1, 1, 1, 2).repeat(N_, 1, 1, 1)
-        # grid = (grid.unsqueeze(0).expand(N_, -1, -1, -1) + 0.5) / scale
-        # wh = torch.ones_like(grid) / scale
-        proposal = torch.cat((grid, wh), -1).view(N_, -1, 4)
-        proposals.append(proposal)
-        _cur += H_ * W_
-    # import ipdb; ipdb.set_trace()
-    output_proposals = torch.cat(proposals, 1)
-    output_proposals_valid = ((output_proposals > 0.01) & (output_proposals < 0.99)).all(
-        -1, keepdim=True
-    )
-    output_proposals = torch.log(output_proposals / (1 - output_proposals))  # unsigmoid
-    output_proposals = output_proposals.masked_fill(memory_padding_mask.unsqueeze(-1), float("inf"))
-    output_proposals = output_proposals.masked_fill(~output_proposals_valid, float("inf"))
-
-    output_memory = memory
-    output_memory = output_memory.masked_fill(memory_padding_mask.unsqueeze(-1), float(0))
-    output_memory = output_memory.masked_fill(~output_proposals_valid, float(0))
-
-    # output_memory = output_memory.masked_fill(memory_padding_mask.unsqueeze(-1), float('inf'))
-    # output_memory = output_memory.masked_fill(~output_proposals_valid, float('inf'))
-
-    return output_memory, output_proposals
-
-
-class RandomBoxPerturber:
-    def __init__(
-        self, x_noise_scale=0.2, y_noise_scale=0.2, w_noise_scale=0.2, h_noise_scale=0.2
-    ) -> None:
-        self.noise_scale = torch.Tensor(
-            [x_noise_scale, y_noise_scale, w_noise_scale, h_noise_scale]
-        )
-
-    def __call__(self, refanchors: Tensor) -> Tensor:
-        nq, bs, query_dim = refanchors.shape
-        device = refanchors.device
-
-        noise_raw = torch.rand_like(refanchors)
-        noise_scale = self.noise_scale.to(device)[:query_dim]
-
-        new_refanchors = refanchors * (1 + (noise_raw - 0.5) * noise_scale)
-        return new_refanchors.clamp_(0, 1)
-
-
-def sigmoid_focal_loss(
-    inputs, targets, num_boxes, alpha: float = 0.25, gamma: float = 2, no_reduction=False
-):
-    """
-    Loss used in RetinaNet for dense detection: https://arxiv.org/abs/1708.02002.
-    Args:
-        inputs: A float tensor of arbitrary shape.
-                The predictions for each example.
-        targets: A float tensor with the same shape as inputs. Stores the binary
-                 classification label for each element in inputs
-                (0 for the negative class and 1 for the positive class).
-        alpha: (optional) Weighting factor in range (0,1) to balance
-                positive vs negative examples. Default = -1 (no weighting).
-        gamma: Exponent of the modulating factor (1 - p_t) to
-               balance easy vs hard examples.
-    Returns:
-        Loss tensor
-    """
-    prob = inputs.sigmoid()
-    ce_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction="none")
-    p_t = prob * targets + (1 - prob) * (1 - targets)
-    loss = ce_loss * ((1 - p_t) ** gamma)
-
-    if alpha >= 0:
-        alpha_t = alpha * targets + (1 - alpha) * (1 - targets)
-        loss = alpha_t * loss
-
-    if no_reduction:
-        return loss
-
-    return loss.mean(1).sum() / num_boxes
-
-
-class MLP(nn.Module):
-    """Very simple multi-layer perceptron (also called FFN)"""
-
-    def __init__(self, input_dim, hidden_dim, output_dim, num_layers):
-        super().__init__()
-        self.num_layers = num_layers
-        h = [hidden_dim] * (num_layers - 1)
-        self.layers = nn.ModuleList(
-            nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim])
-        )
-
-    def forward(self, x):
-        for i, layer in enumerate(self.layers):
-            x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x)
-        return x
-
-
-def _get_activation_fn(activation, d_model=256, batch_dim=0):
-    """Return an activation function given a string"""
-    if activation == "relu":
-        return F.relu
-    if activation == "gelu":
-        return F.gelu
-    if activation == "glu":
-        return F.glu
-    if activation == "prelu":
-        return nn.PReLU()
-    if activation == "selu":
-        return F.selu
-
-    raise RuntimeError(f"activation should be relu/gelu, not {activation}.")
-
-
-def gen_sineembed_for_position(pos_tensor):
-    # n_query, bs, _ = pos_tensor.size()
-    # sineembed_tensor = torch.zeros(n_query, bs, 256)
-    scale = 2 * math.pi
-    dim_t = torch.arange(128, dtype=torch.float32, device=pos_tensor.device)
-    dim_t = 10000 ** (2 * (torch.div(dim_t, 2, rounding_mode='floor')) / 128)
-    x_embed = pos_tensor[:, :, 0] * scale
-    y_embed = pos_tensor[:, :, 1] * scale
-    pos_x = x_embed[:, :, None] / dim_t
-    pos_y = y_embed[:, :, None] / dim_t
-    pos_x = torch.stack((pos_x[:, :, 0::2].sin(), pos_x[:, :, 1::2].cos()), dim=3).flatten(2)
-    pos_y = torch.stack((pos_y[:, :, 0::2].sin(), pos_y[:, :, 1::2].cos()), dim=3).flatten(2)
-    if pos_tensor.size(-1) == 2:
-        pos = torch.cat((pos_y, pos_x), dim=2)
-    elif pos_tensor.size(-1) == 4:
-        w_embed = pos_tensor[:, :, 2] * scale
-        pos_w = w_embed[:, :, None] / dim_t
-        pos_w = torch.stack((pos_w[:, :, 0::2].sin(), pos_w[:, :, 1::2].cos()), dim=3).flatten(2)
-
-        h_embed = pos_tensor[:, :, 3] * scale
-        pos_h = h_embed[:, :, None] / dim_t
-        pos_h = torch.stack((pos_h[:, :, 0::2].sin(), pos_h[:, :, 1::2].cos()), dim=3).flatten(2)
-
-        pos = torch.cat((pos_y, pos_x, pos_w, pos_h), dim=2)
-    else:
-        raise ValueError("Unknown pos_tensor shape(-1):{}".format(pos_tensor.size(-1)))
-    return pos
-
-
-class ContrastiveEmbed(nn.Module):
-    def __init__(self, max_text_len=256):
-        """
-        Args:
-            max_text_len: max length of text.
-        """
-        super().__init__()
-        self.max_text_len = max_text_len
-
-    def forward(self, x, text_dict):
-        """_summary_
-
-        Args:
-            x (_type_): _description_
-            text_dict (_type_): _description_
-            {
-                'encoded_text': encoded_text, # bs, 195, d_model
-                'text_token_mask': text_token_mask, # bs, 195
-                        # True for used tokens. False for padding tokens
-            }
-        Returns:
-            _type_: _description_
-        """
-        assert isinstance(text_dict, dict)
-
-        y = text_dict["encoded_text"]
-        text_token_mask = text_dict["text_token_mask"]
-
-        res = x @ y.transpose(-1, -2)
-        res.masked_fill_(~text_token_mask[:, None, :], float("-inf"))
-
-        # padding to max_text_len
-        new_res = torch.full((*res.shape[:-1], self.max_text_len), float("-inf"), device=res.device)
-        new_res[..., : res.shape[-1]] = res
-
-        return new_res

grounding_dino/groundingdino/models/__init__.py

@@ -1,18 +0,0 @@
-# ------------------------------------------------------------------------
-# Grounding DINO
-# url: https://github.com/IDEA-Research/GroundingDINO
-# Copyright (c) 2023 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-from .GroundingDINO import build_groundingdino
-
-
-def build_model(args):
-    # we use register to maintain models from catdet6 on.
-    from .registry import MODULE_BUILD_FUNCS
-
-    assert args.modelname in MODULE_BUILD_FUNCS._module_dict
-    build_func = MODULE_BUILD_FUNCS.get(args.modelname)
-    model = build_func(args)
-    return model

grounding_dino/groundingdino/models/registry.py

@@ -1,66 +0,0 @@
-# ------------------------------------------------------------------------
-# Grounding DINO
-# url: https://github.com/IDEA-Research/GroundingDINO
-# Copyright (c) 2023 IDEA. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
-# ------------------------------------------------------------------------
-# -*- coding: utf-8 -*-
-# @Author: Yihao Chen
-# @Date:   2021-08-16 16:03:17
-# @Last Modified by:   Shilong Liu
-# @Last Modified time: 2022-01-23 15:26
-# modified from mmcv
-
-import inspect
-from functools import partial
-
-
-class Registry(object):
-    def __init__(self, name):
-        self._name = name
-        self._module_dict = dict()
-
-    def __repr__(self):
-        format_str = self.__class__.__name__ + "(name={}, items={})".format(
-            self._name, list(self._module_dict.keys())
-        )
-        return format_str
-
-    def __len__(self):
-        return len(self._module_dict)
-
-    @property
-    def name(self):
-        return self._name
-
-    @property
-    def module_dict(self):
-        return self._module_dict
-
-    def get(self, key):
-        return self._module_dict.get(key, None)
-
-    def registe_with_name(self, module_name=None, force=False):
-        return partial(self.register, module_name=module_name, force=force)
-
-    def register(self, module_build_function, module_name=None, force=False):
-        """Register a module build function.
-        Args:
-            module (:obj:`nn.Module`): Module to be registered.
-        """
-        if not inspect.isfunction(module_build_function):
-            raise TypeError(
-                "module_build_function must be a function, but got {}".format(
-                    type(module_build_function)
-                )
-            )
-        if module_name is None:
-            module_name = module_build_function.__name__
-        if not force and module_name in self._module_dict:
-            raise KeyError("{} is already registered in {}".format(module_name, self.name))
-        self._module_dict[module_name] = module_build_function
-
-        return module_build_function
-
-
-MODULE_BUILD_FUNCS = Registry("model build functions")

grounding_dino/groundingdino/util/__init__.py

@@ -1 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved

grounding_dino/groundingdino/util/box_ops.py

@@ -1,140 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-"""
-Utilities for bounding box manipulation and GIoU.
-"""
-import torch
-from torchvision.ops.boxes import box_area
-
-
-def box_cxcywh_to_xyxy(x):
-    x_c, y_c, w, h = x.unbind(-1)
-    b = [(x_c - 0.5 * w), (y_c - 0.5 * h), (x_c + 0.5 * w), (y_c + 0.5 * h)]
-    return torch.stack(b, dim=-1)
-
-
-def box_xyxy_to_cxcywh(x):
-    x0, y0, x1, y1 = x.unbind(-1)
-    b = [(x0 + x1) / 2, (y0 + y1) / 2, (x1 - x0), (y1 - y0)]
-    return torch.stack(b, dim=-1)
-
-
-# modified from torchvision to also return the union
-def box_iou(boxes1, boxes2):
-    area1 = box_area(boxes1)
-    area2 = box_area(boxes2)
-
-    # import ipdb; ipdb.set_trace()
-    lt = torch.max(boxes1[:, None, :2], boxes2[:, :2])  # [N,M,2]
-    rb = torch.min(boxes1[:, None, 2:], boxes2[:, 2:])  # [N,M,2]
-
-    wh = (rb - lt).clamp(min=0)  # [N,M,2]
-    inter = wh[:, :, 0] * wh[:, :, 1]  # [N,M]
-
-    union = area1[:, None] + area2 - inter
-
-    iou = inter / (union + 1e-6)
-    return iou, union
-
-
-def generalized_box_iou(boxes1, boxes2):
-    """
-    Generalized IoU from https://giou.stanford.edu/
-
-    The boxes should be in [x0, y0, x1, y1] format
-
-    Returns a [N, M] pairwise matrix, where N = len(boxes1)
-    and M = len(boxes2)
-    """
-    # degenerate boxes gives inf / nan results
-    # so do an early check
-    assert (boxes1[:, 2:] >= boxes1[:, :2]).all()
-    assert (boxes2[:, 2:] >= boxes2[:, :2]).all()
-    # except:
-    #     import ipdb; ipdb.set_trace()
-    iou, union = box_iou(boxes1, boxes2)
-
-    lt = torch.min(boxes1[:, None, :2], boxes2[:, :2])
-    rb = torch.max(boxes1[:, None, 2:], boxes2[:, 2:])
-
-    wh = (rb - lt).clamp(min=0)  # [N,M,2]
-    area = wh[:, :, 0] * wh[:, :, 1]
-
-    return iou - (area - union) / (area + 1e-6)
-
-
-# modified from torchvision to also return the union
-def box_iou_pairwise(boxes1, boxes2):
-    area1 = box_area(boxes1)
-    area2 = box_area(boxes2)
-
-    lt = torch.max(boxes1[:, :2], boxes2[:, :2])  # [N,2]
-    rb = torch.min(boxes1[:, 2:], boxes2[:, 2:])  # [N,2]
-
-    wh = (rb - lt).clamp(min=0)  # [N,2]
-    inter = wh[:, 0] * wh[:, 1]  # [N]
-
-    union = area1 + area2 - inter
-
-    iou = inter / union
-    return iou, union
-
-
-def generalized_box_iou_pairwise(boxes1, boxes2):
-    """
-    Generalized IoU from https://giou.stanford.edu/
-
-    Input:
-        - boxes1, boxes2: N,4
-    Output:
-        - giou: N, 4
-    """
-    # degenerate boxes gives inf / nan results
-    # so do an early check
-    assert (boxes1[:, 2:] >= boxes1[:, :2]).all()
-    assert (boxes2[:, 2:] >= boxes2[:, :2]).all()
-    assert boxes1.shape == boxes2.shape
-    iou, union = box_iou_pairwise(boxes1, boxes2)  # N, 4
-
-    lt = torch.min(boxes1[:, :2], boxes2[:, :2])
-    rb = torch.max(boxes1[:, 2:], boxes2[:, 2:])
-
-    wh = (rb - lt).clamp(min=0)  # [N,2]
-    area = wh[:, 0] * wh[:, 1]
-
-    return iou - (area - union) / area
-
-
-def masks_to_boxes(masks):
-    """Compute the bounding boxes around the provided masks
-
-    The masks should be in format [N, H, W] where N is the number of masks, (H, W) are the spatial dimensions.
-
-    Returns a [N, 4] tensors, with the boxes in xyxy format
-    """
-    if masks.numel() == 0:
-        return torch.zeros((0, 4), device=masks.device)
-
-    h, w = masks.shape[-2:]
-
-    y = torch.arange(0, h, dtype=torch.float)
-    x = torch.arange(0, w, dtype=torch.float)
-    y, x = torch.meshgrid(y, x)
-
-    x_mask = masks * x.unsqueeze(0)
-    x_max = x_mask.flatten(1).max(-1)[0]
-    x_min = x_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0]
-
-    y_mask = masks * y.unsqueeze(0)
-    y_max = y_mask.flatten(1).max(-1)[0]
-    y_min = y_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0]
-
-    return torch.stack([x_min, y_min, x_max, y_max], 1)
-
-
-if __name__ == "__main__":
-    x = torch.rand(5, 4)
-    y = torch.rand(3, 4)
-    iou, union = box_iou(x, y)
-    import ipdb
-
-    ipdb.set_trace()

grounding_dino/groundingdino/util/get_tokenlizer.py

@@ -1,29 +0,0 @@
-from transformers import AutoTokenizer, BertModel, BertTokenizer, RobertaModel, RobertaTokenizerFast
-import os
-
-def get_tokenlizer(text_encoder_type):
-    if not isinstance(text_encoder_type, str):
-        # print("text_encoder_type is not a str")
-        if hasattr(text_encoder_type, "text_encoder_type"):
-            text_encoder_type = text_encoder_type.text_encoder_type
-        elif text_encoder_type.get("text_encoder_type", False):
-            text_encoder_type = text_encoder_type.get("text_encoder_type")
-        elif os.path.isdir(text_encoder_type) and os.path.exists(text_encoder_type):
-            pass
-        else:
-            raise ValueError(
-                "Unknown type of text_encoder_type: {}".format(type(text_encoder_type))
-            )
-    print("final text_encoder_type: {}".format(text_encoder_type))
-
-    tokenizer = AutoTokenizer.from_pretrained(text_encoder_type)
-    return tokenizer
-
-
-def get_pretrained_language_model(text_encoder_type):
-    if text_encoder_type == "bert-base-uncased" or (os.path.isdir(text_encoder_type) and os.path.exists(text_encoder_type)):
-        return BertModel.from_pretrained(text_encoder_type)
-    if text_encoder_type == "roberta-base":
-        return RobertaModel.from_pretrained(text_encoder_type)
-
-    raise ValueError("Unknown text_encoder_type {}".format(text_encoder_type))

grounding_dino/groundingdino/util/inference.py

@@ -1,273 +0,0 @@
-from typing import Tuple, List
-
-import cv2
-import numpy as np
-import supervision as sv
-import torch
-from PIL import Image
-from torchvision.ops import box_convert
-import bisect
-
-import grounding_dino.groundingdino.datasets.transforms as T
-from grounding_dino.groundingdino.models import build_model
-from grounding_dino.groundingdino.util.misc import clean_state_dict
-from grounding_dino.groundingdino.util.slconfig import SLConfig
-from grounding_dino.groundingdino.util.utils import get_phrases_from_posmap
-
-# ----------------------------------------------------------------------------------------------------------------------
-# OLD API
-# ----------------------------------------------------------------------------------------------------------------------
-
-
-def preprocess_caption(caption: str) -> str:
-    result = caption.lower().strip()
-    if result.endswith("."):
-        return result
-    return result + "."
-
-
-def load_model(model_config_path: str, model_checkpoint_path: str, device: str = "cuda"):
-    args = SLConfig.fromfile(model_config_path)
-    args.device = device
-    model = build_model(args)
-    checkpoint = torch.load(model_checkpoint_path, map_location="cpu")
-    model.load_state_dict(clean_state_dict(checkpoint["model"]), strict=False)
-    model.eval()
-    return model
-
-
-def load_image(image_path: str) -> Tuple[np.array, torch.Tensor]:
-    transform = T.Compose(
-        [
-            T.RandomResize([800], max_size=1333),
-            T.ToTensor(),
-            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
-        ]
-    )
-    image_source = Image.open(image_path).convert("RGB")
-    image = np.asarray(image_source)
-    image_transformed, _ = transform(image_source, None)
-    return image, image_transformed
-
-
-def predict(
-        model,
-        image: torch.Tensor,
-        caption: str,
-        box_threshold: float,
-        text_threshold: float,
-        device: str = "cuda",
-        remove_combined: bool = False
-) -> Tuple[torch.Tensor, torch.Tensor, List[str]]:
-    caption = preprocess_caption(caption=caption)
-
-    model = model.to(device)
-    image = image.to(device)
-
-    with torch.no_grad():
-        outputs = model(image[None], captions=[caption])
-
-    prediction_logits = outputs["pred_logits"].cpu().sigmoid()[0]  # prediction_logits.shape = (nq, 256)
-    prediction_boxes = outputs["pred_boxes"].cpu()[0]  # prediction_boxes.shape = (nq, 4)
-
-    mask = prediction_logits.max(dim=1)[0] > box_threshold
-    logits = prediction_logits[mask]  # logits.shape = (n, 256)
-    boxes = prediction_boxes[mask]  # boxes.shape = (n, 4)
-
-    tokenizer = model.tokenizer
-    tokenized = tokenizer(caption)
-    
-    if remove_combined:
-        sep_idx = [i for i in range(len(tokenized['input_ids'])) if tokenized['input_ids'][i] in [101, 102, 1012]]
-        
-        phrases = []
-        for logit in logits:
-            max_idx = logit.argmax()
-            insert_idx = bisect.bisect_left(sep_idx, max_idx)
-            right_idx = sep_idx[insert_idx]
-            left_idx = sep_idx[insert_idx - 1]
-            phrases.append(get_phrases_from_posmap(logit > text_threshold, tokenized, tokenizer, left_idx, right_idx).replace('.', ''))
-    else:
-        phrases = [
-            get_phrases_from_posmap(logit > text_threshold, tokenized, tokenizer).replace('.', '')
-            for logit
-            in logits
-        ]
-
-    return boxes, logits.max(dim=1)[0], phrases
-
-
-def annotate(image_source: np.ndarray, boxes: torch.Tensor, logits: torch.Tensor, phrases: List[str]) -> np.ndarray:
-    """    
-    This function annotates an image with bounding boxes and labels.
-
-    Parameters:
-    image_source (np.ndarray): The source image to be annotated.
-    boxes (torch.Tensor): A tensor containing bounding box coordinates.
-    logits (torch.Tensor): A tensor containing confidence scores for each bounding box.
-    phrases (List[str]): A list of labels for each bounding box.
-
-    Returns:
-    np.ndarray: The annotated image.
-    """
-    h, w, _ = image_source.shape
-    boxes = boxes * torch.Tensor([w, h, w, h])
-    xyxy = box_convert(boxes=boxes, in_fmt="cxcywh", out_fmt="xyxy").numpy()
-    detections = sv.Detections(xyxy=xyxy)
-
-    labels = [
-        f"{phrase} {logit:.2f}"
-        for phrase, logit
-        in zip(phrases, logits)
-    ]
-
-    bbox_annotator = sv.BoxAnnotator(color_lookup=sv.ColorLookup.INDEX)
-    label_annotator = sv.LabelAnnotator(color_lookup=sv.ColorLookup.INDEX)
-    annotated_frame = cv2.cvtColor(image_source, cv2.COLOR_RGB2BGR)
-    annotated_frame = bbox_annotator.annotate(scene=annotated_frame, detections=detections)
-    annotated_frame = label_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels)
-    return annotated_frame
-
-
-# ----------------------------------------------------------------------------------------------------------------------
-# NEW API
-# ----------------------------------------------------------------------------------------------------------------------
-
-
-class Model:
-
-    def __init__(
-        self,
-        model_config_path: str,
-        model_checkpoint_path: str,
-        device: str = "cuda"
-    ):
-        self.model = load_model(
-            model_config_path=model_config_path,
-            model_checkpoint_path=model_checkpoint_path,
-            device=device
-        ).to(device)
-        self.device = device
-
-    def predict_with_caption(
-        self,
-        image: np.ndarray,
-        caption: str,
-        box_threshold: float = 0.35,
-        text_threshold: float = 0.25
-    ) -> Tuple[sv.Detections, List[str]]:
-        """
-        import cv2
-
-        image = cv2.imread(IMAGE_PATH)
-
-        model = Model(model_config_path=CONFIG_PATH, model_checkpoint_path=WEIGHTS_PATH)
-        detections, labels = model.predict_with_caption(
-            image=image,
-            caption=caption,
-            box_threshold=BOX_THRESHOLD,
-            text_threshold=TEXT_THRESHOLD
-        )
-
-        import supervision as sv
-
-        box_annotator = sv.BoxAnnotator()
-        annotated_image = box_annotator.annotate(scene=image, detections=detections, labels=labels)
-        """
-        processed_image = Model.preprocess_image(image_bgr=image).to(self.device)
-        boxes, logits, phrases = predict(
-            model=self.model,
-            image=processed_image,
-            caption=caption,
-            box_threshold=box_threshold,
-            text_threshold=text_threshold, 
-            device=self.device)
-        source_h, source_w, _ = image.shape
-        detections = Model.post_process_result(
-            source_h=source_h,
-            source_w=source_w,
-            boxes=boxes,
-            logits=logits)
-        return detections, phrases
-
-    def predict_with_classes(
-        self,
-        image: np.ndarray,
-        classes: List[str],
-        box_threshold: float,
-        text_threshold: float
-    ) -> sv.Detections:
-        """
-        import cv2
-
-        image = cv2.imread(IMAGE_PATH)
-
-        model = Model(model_config_path=CONFIG_PATH, model_checkpoint_path=WEIGHTS_PATH)
-        detections = model.predict_with_classes(
-            image=image,
-            classes=CLASSES,
-            box_threshold=BOX_THRESHOLD,
-            text_threshold=TEXT_THRESHOLD
-        )
-
-
-        import supervision as sv
-
-        box_annotator = sv.BoxAnnotator()
-        annotated_image = box_annotator.annotate(scene=image, detections=detections)
-        """
-        caption = ". ".join(classes)
-        processed_image = Model.preprocess_image(image_bgr=image).to(self.device)
-        boxes, logits, phrases = predict(
-            model=self.model,
-            image=processed_image,
-            caption=caption,
-            box_threshold=box_threshold,
-            text_threshold=text_threshold,
-            device=self.device)
-        source_h, source_w, _ = image.shape
-        detections = Model.post_process_result(
-            source_h=source_h,
-            source_w=source_w,
-            boxes=boxes,
-            logits=logits)
-        class_id = Model.phrases2classes(phrases=phrases, classes=classes)
-        detections.class_id = class_id
-        return detections
-
-    @staticmethod
-    def preprocess_image(image_bgr: np.ndarray) -> torch.Tensor:
-        transform = T.Compose(
-            [
-                T.RandomResize([800], max_size=1333),
-                T.ToTensor(),
-                T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
-            ]
-        )
-        image_pillow = Image.fromarray(cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB))
-        image_transformed, _ = transform(image_pillow, None)
-        return image_transformed
-
-    @staticmethod
-    def post_process_result(
-            source_h: int,
-            source_w: int,
-            boxes: torch.Tensor,
-            logits: torch.Tensor
-    ) -> sv.Detections:
-        boxes = boxes * torch.Tensor([source_w, source_h, source_w, source_h])
-        xyxy = box_convert(boxes=boxes, in_fmt="cxcywh", out_fmt="xyxy").numpy()
-        confidence = logits.numpy()
-        return sv.Detections(xyxy=xyxy, confidence=confidence)
-
-    @staticmethod
-    def phrases2classes(phrases: List[str], classes: List[str]) -> np.ndarray:
-        class_ids = []
-        for phrase in phrases:
-            for class_ in classes:
-                if class_ in phrase:
-                    class_ids.append(classes.index(class_))
-                    break
-            else:
-                class_ids.append(None)
-        return np.array(class_ids)

grounding_dino/groundingdino/util/logger.py

@@ -1,93 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-import functools
-import logging
-import os
-import sys
-
-from termcolor import colored
-
-
-class _ColorfulFormatter(logging.Formatter):
-    def __init__(self, *args, **kwargs):
-        self._root_name = kwargs.pop("root_name") + "."
-        self._abbrev_name = kwargs.pop("abbrev_name", "")
-        if len(self._abbrev_name):
-            self._abbrev_name = self._abbrev_name + "."
-        super(_ColorfulFormatter, self).__init__(*args, **kwargs)
-
-    def formatMessage(self, record):
-        record.name = record.name.replace(self._root_name, self._abbrev_name)
-        log = super(_ColorfulFormatter, self).formatMessage(record)
-        if record.levelno == logging.WARNING:
-            prefix = colored("WARNING", "red", attrs=["blink"])
-        elif record.levelno == logging.ERROR or record.levelno == logging.CRITICAL:
-            prefix = colored("ERROR", "red", attrs=["blink", "underline"])
-        else:
-            return log
-        return prefix + " " + log
-
-
-# so that calling setup_logger multiple times won't add many handlers
-@functools.lru_cache()
-def setup_logger(output=None, distributed_rank=0, *, color=True, name="imagenet", abbrev_name=None):
-    """
-    Initialize the detectron2 logger and set its verbosity level to "INFO".
-
-    Args:
-        output (str): a file name or a directory to save log. If None, will not save log file.
-            If ends with ".txt" or ".log", assumed to be a file name.
-            Otherwise, logs will be saved to `output/log.txt`.
-        name (str): the root module name of this logger
-
-    Returns:
-        logging.Logger: a logger
-    """
-    logger = logging.getLogger(name)
-    logger.setLevel(logging.DEBUG)
-    logger.propagate = False
-
-    if abbrev_name is None:
-        abbrev_name = name
-
-    plain_formatter = logging.Formatter(
-        "[%(asctime)s.%(msecs)03d]: %(message)s", datefmt="%m/%d %H:%M:%S"
-    )
-    # stdout logging: master only
-    if distributed_rank == 0:
-        ch = logging.StreamHandler(stream=sys.stdout)
-        ch.setLevel(logging.DEBUG)
-        if color:
-            formatter = _ColorfulFormatter(
-                colored("[%(asctime)s.%(msecs)03d]: ", "green") + "%(message)s",
-                datefmt="%m/%d %H:%M:%S",
-                root_name=name,
-                abbrev_name=str(abbrev_name),
-            )
-        else:
-            formatter = plain_formatter
-        ch.setFormatter(formatter)
-        logger.addHandler(ch)
-
-    # file logging: all workers
-    if output is not None:
-        if output.endswith(".txt") or output.endswith(".log"):
-            filename = output
-        else:
-            filename = os.path.join(output, "log.txt")
-        if distributed_rank > 0:
-            filename = filename + f".rank{distributed_rank}"
-        os.makedirs(os.path.dirname(filename), exist_ok=True)
-
-        fh = logging.StreamHandler(_cached_log_stream(filename))
-        fh.setLevel(logging.DEBUG)
-        fh.setFormatter(plain_formatter)
-        logger.addHandler(fh)
-
-    return logger
-
-
-# cache the opened file object, so that different calls to `setup_logger`
-# with the same file name can safely write to the same file.
-@functools.lru_cache(maxsize=None)
-def _cached_log_stream(filename):
-    return open(filename, "a")

grounding_dino/groundingdino/util/slconfig.py

@@ -1,427 +0,0 @@
-# ==========================================================
-# Modified from mmcv
-# ==========================================================
-import ast
-import os
-import os.path as osp
-import shutil
-import sys
-import tempfile
-from argparse import Action
-from importlib import import_module
-
-from addict import Dict
-from yapf.yapflib.yapf_api import FormatCode
-
-BASE_KEY = "_base_"
-DELETE_KEY = "_delete_"
-RESERVED_KEYS = ["filename", "text", "pretty_text", "get", "dump", "merge_from_dict"]
-
-
-def check_file_exist(filename, msg_tmpl='file "{}" does not exist'):
-    if not osp.isfile(filename):
-        raise FileNotFoundError(msg_tmpl.format(filename))
-
-
-class ConfigDict(Dict):
-    def __missing__(self, name):
-        raise KeyError(name)
-
-    def __getattr__(self, name):
-        try:
-            value = super(ConfigDict, self).__getattr__(name)
-        except KeyError:
-            ex = AttributeError(f"'{self.__class__.__name__}' object has no " f"attribute '{name}'")
-        except Exception as e:
-            ex = e
-        else:
-            return value
-        raise ex
-
-
-class SLConfig(object):
-    """
-    config files.
-    only support .py file as config now.
-
-    ref: mmcv.utils.config
-
-    Example:
-        >>> cfg = Config(dict(a=1, b=dict(b1=[0, 1])))
-        >>> cfg.a
-        1
-        >>> cfg.b
-        {'b1': [0, 1]}
-        >>> cfg.b.b1
-        [0, 1]
-        >>> cfg = Config.fromfile('tests/data/config/a.py')
-        >>> cfg.filename
-        "/home/kchen/projects/mmcv/tests/data/config/a.py"
-        >>> cfg.item4
-        'test'
-        >>> cfg
-        "Config [path: /home/kchen/projects/mmcv/tests/data/config/a.py]: "
-        "{'item1': [1, 2], 'item2': {'a': 0}, 'item3': True, 'item4': 'test'}"
-    """
-
-    @staticmethod
-    def _validate_py_syntax(filename):
-        with open(filename) as f:
-            content = f.read()
-        try:
-            ast.parse(content)
-        except SyntaxError:
-            raise SyntaxError("There are syntax errors in config " f"file {filename}")
-
-    @staticmethod
-    def _file2dict(filename):
-        filename = osp.abspath(osp.expanduser(filename))
-        check_file_exist(filename)
-        if filename.lower().endswith(".py"):
-            with tempfile.TemporaryDirectory() as temp_config_dir:
-                temp_config_file = tempfile.NamedTemporaryFile(dir=temp_config_dir, suffix=".py")
-                temp_config_name = osp.basename(temp_config_file.name)
-                if os.name == 'nt':
-                    temp_config_file.close()
-                shutil.copyfile(filename, osp.join(temp_config_dir, temp_config_name))
-                temp_module_name = osp.splitext(temp_config_name)[0]
-                sys.path.insert(0, temp_config_dir)
-                SLConfig._validate_py_syntax(filename)
-                mod = import_module(temp_module_name)
-                sys.path.pop(0)
-                cfg_dict = {
-                    name: value for name, value in mod.__dict__.items() if not name.startswith("__")
-                }
-                # delete imported module
-                del sys.modules[temp_module_name]
-                # close temp file
-                temp_config_file.close()
-        elif filename.lower().endswith((".yml", ".yaml", ".json")):
-            from .slio import slload
-
-            cfg_dict = slload(filename)
-        else:
-            raise IOError("Only py/yml/yaml/json type are supported now!")
-
-        cfg_text = filename + "\n"
-        with open(filename, "r") as f:
-            cfg_text += f.read()
-
-        # parse the base file
-        if BASE_KEY in cfg_dict:
-            cfg_dir = osp.dirname(filename)
-            base_filename = cfg_dict.pop(BASE_KEY)
-            base_filename = base_filename if isinstance(base_filename, list) else [base_filename]
-
-            cfg_dict_list = list()
-            cfg_text_list = list()
-            for f in base_filename:
-                _cfg_dict, _cfg_text = SLConfig._file2dict(osp.join(cfg_dir, f))
-                cfg_dict_list.append(_cfg_dict)
-                cfg_text_list.append(_cfg_text)
-
-            base_cfg_dict = dict()
-            for c in cfg_dict_list:
-                if len(base_cfg_dict.keys() & c.keys()) > 0:
-                    raise KeyError("Duplicate key is not allowed among bases")
-                    # TODO Allow the duplicate key while warnning user
-                base_cfg_dict.update(c)
-
-            base_cfg_dict = SLConfig._merge_a_into_b(cfg_dict, base_cfg_dict)
-            cfg_dict = base_cfg_dict
-
-            # merge cfg_text
-            cfg_text_list.append(cfg_text)
-            cfg_text = "\n".join(cfg_text_list)
-
-        return cfg_dict, cfg_text
-
-    @staticmethod
-    def _merge_a_into_b(a, b):
-        """merge dict `a` into dict `b` (non-inplace).
-            values in `a` will overwrite `b`.
-            copy first to avoid inplace modification
-
-        Args:
-            a ([type]): [description]
-            b ([type]): [description]
-
-        Returns:
-            [dict]: [description]
-        """
-        # import ipdb; ipdb.set_trace()
-        if not isinstance(a, dict):
-            return a
-
-        b = b.copy()
-        for k, v in a.items():
-            if isinstance(v, dict) and k in b and not v.pop(DELETE_KEY, False):
-
-                if not isinstance(b[k], dict) and not isinstance(b[k], list):
-                    # if :
-                    # import ipdb; ipdb.set_trace()
-                    raise TypeError(
-                        f"{k}={v} in child config cannot inherit from base "
-                        f"because {k} is a dict in the child config but is of "
-                        f"type {type(b[k])} in base config. You may set "
-                        f"`{DELETE_KEY}=True` to ignore the base config"
-                    )
-                b[k] = SLConfig._merge_a_into_b(v, b[k])
-            elif isinstance(b, list):
-                try:
-                    _ = int(k)
-                except:
-                    raise TypeError(
-                        f"b is a list, " f"index {k} should be an int when input but {type(k)}"
-                    )
-                b[int(k)] = SLConfig._merge_a_into_b(v, b[int(k)])
-            else:
-                b[k] = v
-
-        return b
-
-    @staticmethod
-    def fromfile(filename):
-        cfg_dict, cfg_text = SLConfig._file2dict(filename)
-        return SLConfig(cfg_dict, cfg_text=cfg_text, filename=filename)
-
-    def __init__(self, cfg_dict=None, cfg_text=None, filename=None):
-        if cfg_dict is None:
-            cfg_dict = dict()
-        elif not isinstance(cfg_dict, dict):
-            raise TypeError("cfg_dict must be a dict, but " f"got {type(cfg_dict)}")
-        for key in cfg_dict:
-            if key in RESERVED_KEYS:
-                raise KeyError(f"{key} is reserved for config file")
-
-        super(SLConfig, self).__setattr__("_cfg_dict", ConfigDict(cfg_dict))
-        super(SLConfig, self).__setattr__("_filename", filename)
-        if cfg_text:
-            text = cfg_text
-        elif filename:
-            with open(filename, "r") as f:
-                text = f.read()
-        else:
-            text = ""
-        super(SLConfig, self).__setattr__("_text", text)
-
-    @property
-    def filename(self):
-        return self._filename
-
-    @property
-    def text(self):
-        return self._text
-
-    @property
-    def pretty_text(self):
-
-        indent = 4
-
-        def _indent(s_, num_spaces):
-            s = s_.split("\n")
-            if len(s) == 1:
-                return s_
-            first = s.pop(0)
-            s = [(num_spaces * " ") + line for line in s]
-            s = "\n".join(s)
-            s = first + "\n" + s
-            return s
-
-        def _format_basic_types(k, v, use_mapping=False):
-            if isinstance(v, str):
-                v_str = f"'{v}'"
-            else:
-                v_str = str(v)
-
-            if use_mapping:
-                k_str = f"'{k}'" if isinstance(k, str) else str(k)
-                attr_str = f"{k_str}: {v_str}"
-            else:
-                attr_str = f"{str(k)}={v_str}"
-            attr_str = _indent(attr_str, indent)
-
-            return attr_str
-
-        def _format_list(k, v, use_mapping=False):
-            # check if all items in the list are dict
-            if all(isinstance(_, dict) for _ in v):
-                v_str = "[\n"
-                v_str += "\n".join(
-                    f"dict({_indent(_format_dict(v_), indent)})," for v_ in v
-                ).rstrip(",")
-                if use_mapping:
-                    k_str = f"'{k}'" if isinstance(k, str) else str(k)
-                    attr_str = f"{k_str}: {v_str}"
-                else:
-                    attr_str = f"{str(k)}={v_str}"
-                attr_str = _indent(attr_str, indent) + "]"
-            else:
-                attr_str = _format_basic_types(k, v, use_mapping)
-            return attr_str
-
-        def _contain_invalid_identifier(dict_str):
-            contain_invalid_identifier = False
-            for key_name in dict_str:
-                contain_invalid_identifier |= not str(key_name).isidentifier()
-            return contain_invalid_identifier
-
-        def _format_dict(input_dict, outest_level=False):
-            r = ""
-            s = []
-
-            use_mapping = _contain_invalid_identifier(input_dict)
-            if use_mapping:
-                r += "{"
-            for idx, (k, v) in enumerate(input_dict.items()):
-                is_last = idx >= len(input_dict) - 1
-                end = "" if outest_level or is_last else ","
-                if isinstance(v, dict):
-                    v_str = "\n" + _format_dict(v)
-                    if use_mapping:
-                        k_str = f"'{k}'" if isinstance(k, str) else str(k)
-                        attr_str = f"{k_str}: dict({v_str}"
-                    else:
-                        attr_str = f"{str(k)}=dict({v_str}"
-                    attr_str = _indent(attr_str, indent) + ")" + end
-                elif isinstance(v, list):
-                    attr_str = _format_list(k, v, use_mapping) + end
-                else:
-                    attr_str = _format_basic_types(k, v, use_mapping) + end
-
-                s.append(attr_str)
-            r += "\n".join(s)
-            if use_mapping:
-                r += "}"
-            return r
-
-        cfg_dict = self._cfg_dict.to_dict()
-        text = _format_dict(cfg_dict, outest_level=True)
-        # copied from setup.cfg
-        yapf_style = dict(
-            based_on_style="pep8",
-            blank_line_before_nested_class_or_def=True,
-            split_before_expression_after_opening_paren=True,
-        )
-        text, _ = FormatCode(text, style_config=yapf_style, verify=True)
-
-        return text
-
-    def __repr__(self):
-        return f"Config (path: {self.filename}): {self._cfg_dict.__repr__()}"
-
-    def __len__(self):
-        return len(self._cfg_dict)
-
-    def __getattr__(self, name):
-        # # debug
-        # print('+'*15)
-        # print('name=%s' % name)
-        # print("addr:", id(self))
-        # # print('type(self):', type(self))
-        # print(self.__dict__)
-        # print('+'*15)
-        # if self.__dict__ == {}:
-        #     raise ValueError
-
-        return getattr(self._cfg_dict, name)
-
-    def __getitem__(self, name):
-        return self._cfg_dict.__getitem__(name)
-
-    def __setattr__(self, name, value):
-        if isinstance(value, dict):
-            value = ConfigDict(value)
-        self._cfg_dict.__setattr__(name, value)
-
-    def __setitem__(self, name, value):
-        if isinstance(value, dict):
-            value = ConfigDict(value)
-        self._cfg_dict.__setitem__(name, value)
-
-    def __iter__(self):
-        return iter(self._cfg_dict)
-
-    def dump(self, file=None):
-        # import ipdb; ipdb.set_trace()
-        if file is None:
-            return self.pretty_text
-        else:
-            with open(file, "w") as f:
-                f.write(self.pretty_text)
-
-    def merge_from_dict(self, options):
-        """Merge list into cfg_dict
-
-        Merge the dict parsed by MultipleKVAction into this cfg.
-
-        Examples:
-            >>> options = {'model.backbone.depth': 50,
-            ...            'model.backbone.with_cp':True}
-            >>> cfg = Config(dict(model=dict(backbone=dict(type='ResNet'))))
-            >>> cfg.merge_from_dict(options)
-            >>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict')
-            >>> assert cfg_dict == dict(
-            ...     model=dict(backbone=dict(depth=50, with_cp=True)))
-
-        Args:
-            options (dict): dict of configs to merge from.
-        """
-        option_cfg_dict = {}
-        for full_key, v in options.items():
-            d = option_cfg_dict
-            key_list = full_key.split(".")
-            for subkey in key_list[:-1]:
-                d.setdefault(subkey, ConfigDict())
-                d = d[subkey]
-            subkey = key_list[-1]
-            d[subkey] = v
-
-        cfg_dict = super(SLConfig, self).__getattribute__("_cfg_dict")
-        super(SLConfig, self).__setattr__(
-            "_cfg_dict", SLConfig._merge_a_into_b(option_cfg_dict, cfg_dict)
-        )
-
-    # for multiprocess
-    def __setstate__(self, state):
-        self.__init__(state)
-
-    def copy(self):
-        return SLConfig(self._cfg_dict.copy())
-
-    def deepcopy(self):
-        return SLConfig(self._cfg_dict.deepcopy())
-
-
-class DictAction(Action):
-    """
-    argparse action to split an argument into KEY=VALUE form
-    on the first = and append to a dictionary. List options should
-    be passed as comma separated values, i.e KEY=V1,V2,V3
-    """
-
-    @staticmethod
-    def _parse_int_float_bool(val):
-        try:
-            return int(val)
-        except ValueError:
-            pass
-        try:
-            return float(val)
-        except ValueError:
-            pass
-        if val.lower() in ["true", "false"]:
-            return True if val.lower() == "true" else False
-        if val.lower() in ["none", "null"]:
-            return None
-        return val
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        options = {}
-        for kv in values:
-            key, val = kv.split("=", maxsplit=1)
-            val = [self._parse_int_float_bool(v) for v in val.split(",")]
-            if len(val) == 1:
-                val = val[0]
-            options[key] = val
-        setattr(namespace, self.dest, options)

grounding_dino/groundingdino/util/slio.py

@@ -1,177 +0,0 @@
-# ==========================================================
-# Modified from mmcv
-# ==========================================================
-
-import json
-import pickle
-from abc import ABCMeta, abstractmethod
-from pathlib import Path
-
-import yaml
-
-try:
-    from yaml import CLoader as Loader, CDumper as Dumper
-except ImportError:
-    from yaml import Loader, Dumper
-
-
-# ===========================
-# Rigister handler
-# ===========================
-
-
-class BaseFileHandler(metaclass=ABCMeta):
-    @abstractmethod
-    def load_from_fileobj(self, file, **kwargs):
-        pass
-
-    @abstractmethod
-    def dump_to_fileobj(self, obj, file, **kwargs):
-        pass
-
-    @abstractmethod
-    def dump_to_str(self, obj, **kwargs):
-        pass
-
-    def load_from_path(self, filepath, mode="r", **kwargs):
-        with open(filepath, mode) as f:
-            return self.load_from_fileobj(f, **kwargs)
-
-    def dump_to_path(self, obj, filepath, mode="w", **kwargs):
-        with open(filepath, mode) as f:
-            self.dump_to_fileobj(obj, f, **kwargs)
-
-
-class JsonHandler(BaseFileHandler):
-    def load_from_fileobj(self, file):
-        return json.load(file)
-
-    def dump_to_fileobj(self, obj, file, **kwargs):
-        json.dump(obj, file, **kwargs)
-
-    def dump_to_str(self, obj, **kwargs):
-        return json.dumps(obj, **kwargs)
-
-
-class PickleHandler(BaseFileHandler):
-    def load_from_fileobj(self, file, **kwargs):
-        return pickle.load(file, **kwargs)
-
-    def load_from_path(self, filepath, **kwargs):
-        return super(PickleHandler, self).load_from_path(filepath, mode="rb", **kwargs)
-
-    def dump_to_str(self, obj, **kwargs):
-        kwargs.setdefault("protocol", 2)
-        return pickle.dumps(obj, **kwargs)
-
-    def dump_to_fileobj(self, obj, file, **kwargs):
-        kwargs.setdefault("protocol", 2)
-        pickle.dump(obj, file, **kwargs)
-
-    def dump_to_path(self, obj, filepath, **kwargs):
-        super(PickleHandler, self).dump_to_path(obj, filepath, mode="wb", **kwargs)
-
-
-class YamlHandler(BaseFileHandler):
-    def load_from_fileobj(self, file, **kwargs):
-        kwargs.setdefault("Loader", Loader)
-        return yaml.load(file, **kwargs)
-
-    def dump_to_fileobj(self, obj, file, **kwargs):
-        kwargs.setdefault("Dumper", Dumper)
-        yaml.dump(obj, file, **kwargs)
-
-    def dump_to_str(self, obj, **kwargs):
-        kwargs.setdefault("Dumper", Dumper)
-        return yaml.dump(obj, **kwargs)
-
-
-file_handlers = {
-    "json": JsonHandler(),
-    "yaml": YamlHandler(),
-    "yml": YamlHandler(),
-    "pickle": PickleHandler(),
-    "pkl": PickleHandler(),
-}
-
-# ===========================
-# load and dump
-# ===========================
-
-
-def is_str(x):
-    """Whether the input is an string instance.
-
-    Note: This method is deprecated since python 2 is no longer supported.
-    """
-    return isinstance(x, str)
-
-
-def slload(file, file_format=None, **kwargs):
-    """Load data from json/yaml/pickle files.
-
-    This method provides a unified api for loading data from serialized files.
-
-    Args:
-        file (str or :obj:`Path` or file-like object): Filename or a file-like
-            object.
-        file_format (str, optional): If not specified, the file format will be
-            inferred from the file extension, otherwise use the specified one.
-            Currently supported formats include "json", "yaml/yml" and
-            "pickle/pkl".
-
-    Returns:
-        The content from the file.
-    """
-    if isinstance(file, Path):
-        file = str(file)
-    if file_format is None and is_str(file):
-        file_format = file.split(".")[-1]
-    if file_format not in file_handlers:
-        raise TypeError(f"Unsupported format: {file_format}")
-
-    handler = file_handlers[file_format]
-    if is_str(file):
-        obj = handler.load_from_path(file, **kwargs)
-    elif hasattr(file, "read"):
-        obj = handler.load_from_fileobj(file, **kwargs)
-    else:
-        raise TypeError('"file" must be a filepath str or a file-object')
-    return obj
-
-
-def sldump(obj, file=None, file_format=None, **kwargs):
-    """Dump data to json/yaml/pickle strings or files.
-
-    This method provides a unified api for dumping data as strings or to files,
-    and also supports custom arguments for each file format.
-
-    Args:
-        obj (any): The python object to be dumped.
-        file (str or :obj:`Path` or file-like object, optional): If not
-            specified, then the object is dump to a str, otherwise to a file
-            specified by the filename or file-like object.
-        file_format (str, optional): Same as :func:`load`.
-
-    Returns:
-        bool: True for success, False otherwise.
-    """
-    if isinstance(file, Path):
-        file = str(file)
-    if file_format is None:
-        if is_str(file):
-            file_format = file.split(".")[-1]
-        elif file is None:
-            raise ValueError("file_format must be specified since file is None")
-    if file_format not in file_handlers:
-        raise TypeError(f"Unsupported format: {file_format}")
-
-    handler = file_handlers[file_format]
-    if file is None:
-        return handler.dump_to_str(obj, **kwargs)
-    elif is_str(file):
-        handler.dump_to_path(obj, file, **kwargs)
-    elif hasattr(file, "write"):
-        handler.dump_to_fileobj(obj, file, **kwargs)
-    else:
-        raise TypeError('"file" must be a filename str or a file-object')

grounding_dino/groundingdino/util/time_counter.py

@@ -1,62 +0,0 @@
-import json
-import time
-
-
-class TimeCounter:
-    def __init__(self) -> None:
-        pass
-
-    def clear(self):
-        self.timedict = {}
-        self.basetime = time.perf_counter()
-
-    def timeit(self, name):
-        nowtime = time.perf_counter() - self.basetime
-        self.timedict[name] = nowtime
-        self.basetime = time.perf_counter()
-
-
-class TimeHolder:
-    def __init__(self) -> None:
-        self.timedict = {}
-
-    def update(self, _timedict: dict):
-        for k, v in _timedict.items():
-            if k not in self.timedict:
-                self.timedict[k] = AverageMeter(name=k, val_only=True)
-            self.timedict[k].update(val=v)
-
-    def final_res(self):
-        return {k: v.avg for k, v in self.timedict.items()}
-
-    def __str__(self):
-        return json.dumps(self.final_res(), indent=2)
-
-
-class AverageMeter(object):
-    """Computes and stores the average and current value"""
-
-    def __init__(self, name, fmt=":f", val_only=False):
-        self.name = name
-        self.fmt = fmt
-        self.val_only = val_only
-        self.reset()
-
-    def reset(self):
-        self.val = 0
-        self.avg = 0
-        self.sum = 0
-        self.count = 0
-
-    def update(self, val, n=1):
-        self.val = val
-        self.sum += val * n
-        self.count += n
-        self.avg = self.sum / self.count
-
-    def __str__(self):
-        if self.val_only:
-            fmtstr = "{name} {val" + self.fmt + "}"
-        else:
-            fmtstr = "{name} {val" + self.fmt + "} ({avg" + self.fmt + "})"
-        return fmtstr.format(**self.__dict__)

grounding_dino/groundingdino/util/utils.py

@@ -1,610 +0,0 @@
-import argparse
-import json
-import warnings
-from collections import OrderedDict
-from copy import deepcopy
-from typing import Any, Dict, List
-
-import numpy as np
-import torch
-from transformers import AutoTokenizer
-
-from grounding_dino.groundingdino.util.slconfig import SLConfig
-
-
-def slprint(x, name="x"):
-    if isinstance(x, (torch.Tensor, np.ndarray)):
-        print(f"{name}.shape:", x.shape)
-    elif isinstance(x, (tuple, list)):
-        print("type x:", type(x))
-        for i in range(min(10, len(x))):
-            slprint(x[i], f"{name}[{i}]")
-    elif isinstance(x, dict):
-        for k, v in x.items():
-            slprint(v, f"{name}[{k}]")
-    else:
-        print(f"{name}.type:", type(x))
-
-
-def clean_state_dict(state_dict):
-    new_state_dict = OrderedDict()
-    for k, v in state_dict.items():
-        if k[:7] == "module.":
-            k = k[7:]  # remove `module.`
-        new_state_dict[k] = v
-    return new_state_dict
-
-
-def renorm(
-    img: torch.FloatTensor, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
-) -> torch.FloatTensor:
-    # img: tensor(3,H,W) or tensor(B,3,H,W)
-    # return: same as img
-    assert img.dim() == 3 or img.dim() == 4, "img.dim() should be 3 or 4 but %d" % img.dim()
-    if img.dim() == 3:
-        assert img.size(0) == 3, 'img.size(0) shoule be 3 but "%d". (%s)' % (
-            img.size(0),
-            str(img.size()),
-        )
-        img_perm = img.permute(1, 2, 0)
-        mean = torch.Tensor(mean)
-        std = torch.Tensor(std)
-        img_res = img_perm * std + mean
-        return img_res.permute(2, 0, 1)
-    else:  # img.dim() == 4
-        assert img.size(1) == 3, 'img.size(1) shoule be 3 but "%d". (%s)' % (
-            img.size(1),
-            str(img.size()),
-        )
-        img_perm = img.permute(0, 2, 3, 1)
-        mean = torch.Tensor(mean)
-        std = torch.Tensor(std)
-        img_res = img_perm * std + mean
-        return img_res.permute(0, 3, 1, 2)
-
-
-class CocoClassMapper:
-    def __init__(self) -> None:
-        self.category_map_str = {
-            "1": 1,
-            "2": 2,
-            "3": 3,
-            "4": 4,
-            "5": 5,
-            "6": 6,
-            "7": 7,
-            "8": 8,
-            "9": 9,
-            "10": 10,
-            "11": 11,
-            "13": 12,
-            "14": 13,
-            "15": 14,
-            "16": 15,
-            "17": 16,
-            "18": 17,
-            "19": 18,
-            "20": 19,
-            "21": 20,
-            "22": 21,
-            "23": 22,
-            "24": 23,
-            "25": 24,
-            "27": 25,
-            "28": 26,
-            "31": 27,
-            "32": 28,
-            "33": 29,
-            "34": 30,
-            "35": 31,
-            "36": 32,
-            "37": 33,
-            "38": 34,
-            "39": 35,
-            "40": 36,
-            "41": 37,
-            "42": 38,
-            "43": 39,
-            "44": 40,
-            "46": 41,
-            "47": 42,
-            "48": 43,
-            "49": 44,
-            "50": 45,
-            "51": 46,
-            "52": 47,
-            "53": 48,
-            "54": 49,
-            "55": 50,
-            "56": 51,
-            "57": 52,
-            "58": 53,
-            "59": 54,
-            "60": 55,
-            "61": 56,
-            "62": 57,
-            "63": 58,
-            "64": 59,
-            "65": 60,
-            "67": 61,
-            "70": 62,
-            "72": 63,
-            "73": 64,
-            "74": 65,
-            "75": 66,
-            "76": 67,
-            "77": 68,
-            "78": 69,
-            "79": 70,
-            "80": 71,
-            "81": 72,
-            "82": 73,
-            "84": 74,
-            "85": 75,
-            "86": 76,
-            "87": 77,
-            "88": 78,
-            "89": 79,
-            "90": 80,
-        }
-        self.origin2compact_mapper = {int(k): v - 1 for k, v in self.category_map_str.items()}
-        self.compact2origin_mapper = {int(v - 1): int(k) for k, v in self.category_map_str.items()}
-
-    def origin2compact(self, idx):
-        return self.origin2compact_mapper[int(idx)]
-
-    def compact2origin(self, idx):
-        return self.compact2origin_mapper[int(idx)]
-
-
-def to_device(item, device):
-    if isinstance(item, torch.Tensor):
-        return item.to(device)
-    elif isinstance(item, list):
-        return [to_device(i, device) for i in item]
-    elif isinstance(item, dict):
-        return {k: to_device(v, device) for k, v in item.items()}
-    else:
-        raise NotImplementedError(
-            "Call Shilong if you use other containers! type: {}".format(type(item))
-        )
-
-
-#
-def get_gaussian_mean(x, axis, other_axis, softmax=True):
-    """
-
-    Args:
-        x (float): Input images(BxCxHxW)
-        axis (int): The index for weighted mean
-        other_axis (int): The other index
-
-    Returns: weighted index for axis, BxC
-
-    """
-    mat2line = torch.sum(x, axis=other_axis)
-    # mat2line = mat2line / mat2line.mean() * 10
-    if softmax:
-        u = torch.softmax(mat2line, axis=2)
-    else:
-        u = mat2line / (mat2line.sum(2, keepdim=True) + 1e-6)
-    size = x.shape[axis]
-    ind = torch.linspace(0, 1, size).to(x.device)
-    batch = x.shape[0]
-    channel = x.shape[1]
-    index = ind.repeat([batch, channel, 1])
-    mean_position = torch.sum(index * u, dim=2)
-    return mean_position
-
-
-def get_expected_points_from_map(hm, softmax=True):
-    """get_gaussian_map_from_points
-        B,C,H,W -> B,N,2 float(0, 1) float(0, 1)
-        softargmax function
-
-    Args:
-        hm (float): Input images(BxCxHxW)
-
-    Returns:
-        weighted index for axis, BxCx2. float between 0 and 1.
-
-    """
-    # hm = 10*hm
-    B, C, H, W = hm.shape
-    y_mean = get_gaussian_mean(hm, 2, 3, softmax=softmax)  # B,C
-    x_mean = get_gaussian_mean(hm, 3, 2, softmax=softmax)  # B,C
-    # return torch.cat((x_mean.unsqueeze(-1), y_mean.unsqueeze(-1)), 2)
-    return torch.stack([x_mean, y_mean], dim=2)
-
-
-# Positional encoding (section 5.1)
-# borrow from nerf
-class Embedder:
-    def __init__(self, **kwargs):
-        self.kwargs = kwargs
-        self.create_embedding_fn()
-
-    def create_embedding_fn(self):
-        embed_fns = []
-        d = self.kwargs["input_dims"]
-        out_dim = 0
-        if self.kwargs["include_input"]:
-            embed_fns.append(lambda x: x)
-            out_dim += d
-
-        max_freq = self.kwargs["max_freq_log2"]
-        N_freqs = self.kwargs["num_freqs"]
-
-        if self.kwargs["log_sampling"]:
-            freq_bands = 2.0 ** torch.linspace(0.0, max_freq, steps=N_freqs)
-        else:
-            freq_bands = torch.linspace(2.0**0.0, 2.0**max_freq, steps=N_freqs)
-
-        for freq in freq_bands:
-            for p_fn in self.kwargs["periodic_fns"]:
-                embed_fns.append(lambda x, p_fn=p_fn, freq=freq: p_fn(x * freq))
-                out_dim += d
-
-        self.embed_fns = embed_fns
-        self.out_dim = out_dim
-
-    def embed(self, inputs):
-        return torch.cat([fn(inputs) for fn in self.embed_fns], -1)
-
-
-def get_embedder(multires, i=0):
-    import torch.nn as nn
-
-    if i == -1:
-        return nn.Identity(), 3
-
-    embed_kwargs = {
-        "include_input": True,
-        "input_dims": 3,
-        "max_freq_log2": multires - 1,
-        "num_freqs": multires,
-        "log_sampling": True,
-        "periodic_fns": [torch.sin, torch.cos],
-    }
-
-    embedder_obj = Embedder(**embed_kwargs)
-    embed = lambda x, eo=embedder_obj: eo.embed(x)
-    return embed, embedder_obj.out_dim
-
-
-class APOPMeter:
-    def __init__(self) -> None:
-        self.tp = 0
-        self.fp = 0
-        self.tn = 0
-        self.fn = 0
-
-    def update(self, pred, gt):
-        """
-        Input:
-            pred, gt: Tensor()
-        """
-        assert pred.shape == gt.shape
-        self.tp += torch.logical_and(pred == 1, gt == 1).sum().item()
-        self.fp += torch.logical_and(pred == 1, gt == 0).sum().item()
-        self.tn += torch.logical_and(pred == 0, gt == 0).sum().item()
-        self.tn += torch.logical_and(pred == 1, gt == 0).sum().item()
-
-    def update_cm(self, tp, fp, tn, fn):
-        self.tp += tp
-        self.fp += fp
-        self.tn += tn
-        self.tn += fn
-
-
-def inverse_sigmoid(x, eps=1e-5):
-    x = x.clamp(min=0, max=1)
-    x1 = x.clamp(min=eps)
-    x2 = (1 - x).clamp(min=eps)
-    return torch.log(x1 / x2)
-
-
-def get_raw_dict(args):
-    """
-    return the dicf contained in args.
-
-    e.g:
-        >>> with open(path, 'w') as f:
-                json.dump(get_raw_dict(args), f, indent=2)
-    """
-    if isinstance(args, argparse.Namespace):
-        return vars(args)
-    elif isinstance(args, dict):
-        return args
-    elif isinstance(args, SLConfig):
-        return args._cfg_dict
-    else:
-        raise NotImplementedError("Unknown type {}".format(type(args)))
-
-
-def stat_tensors(tensor):
-    assert tensor.dim() == 1
-    tensor_sm = tensor.softmax(0)
-    entropy = (tensor_sm * torch.log(tensor_sm + 1e-9)).sum()
-
-    return {
-        "max": tensor.max(),
-        "min": tensor.min(),
-        "mean": tensor.mean(),
-        "var": tensor.var(),
-        "std": tensor.var() ** 0.5,
-        "entropy": entropy,
-    }
-
-
-class NiceRepr:
-    """Inherit from this class and define ``__nice__`` to "nicely" print your
-    objects.
-
-    Defines ``__str__`` and ``__repr__`` in terms of ``__nice__`` function
-    Classes that inherit from :class:`NiceRepr` should redefine ``__nice__``.
-    If the inheriting class has a ``__len__``, method then the default
-    ``__nice__`` method will return its length.
-
-    Example:
-        >>> class Foo(NiceRepr):
-        ...    def __nice__(self):
-        ...        return 'info'
-        >>> foo = Foo()
-        >>> assert str(foo) == '<Foo(info)>'
-        >>> assert repr(foo).startswith('<Foo(info) at ')
-
-    Example:
-        >>> class Bar(NiceRepr):
-        ...    pass
-        >>> bar = Bar()
-        >>> import pytest
-        >>> with pytest.warns(None) as record:
-        >>>     assert 'object at' in str(bar)
-        >>>     assert 'object at' in repr(bar)
-
-    Example:
-        >>> class Baz(NiceRepr):
-        ...    def __len__(self):
-        ...        return 5
-        >>> baz = Baz()
-        >>> assert str(baz) == '<Baz(5)>'
-    """
-
-    def __nice__(self):
-        """str: a "nice" summary string describing this module"""
-        if hasattr(self, "__len__"):
-            # It is a common pattern for objects to use __len__ in __nice__
-            # As a convenience we define a default __nice__ for these objects
-            return str(len(self))
-        else:
-            # In all other cases force the subclass to overload __nice__
-            raise NotImplementedError(f"Define the __nice__ method for {self.__class__!r}")
-
-    def __repr__(self):
-        """str: the string of the module"""
-        try:
-            nice = self.__nice__()
-            classname = self.__class__.__name__
-            return f"<{classname}({nice}) at {hex(id(self))}>"
-        except NotImplementedError as ex:
-            warnings.warn(str(ex), category=RuntimeWarning)
-            return object.__repr__(self)
-
-    def __str__(self):
-        """str: the string of the module"""
-        try:
-            classname = self.__class__.__name__
-            nice = self.__nice__()
-            return f"<{classname}({nice})>"
-        except NotImplementedError as ex:
-            warnings.warn(str(ex), category=RuntimeWarning)
-            return object.__repr__(self)
-
-
-def ensure_rng(rng=None):
-    """Coerces input into a random number generator.
-
-    If the input is None, then a global random state is returned.
-
-    If the input is a numeric value, then that is used as a seed to construct a
-    random state. Otherwise the input is returned as-is.
-
-    Adapted from [1]_.
-
-    Args:
-        rng (int | numpy.random.RandomState | None):
-            if None, then defaults to the global rng. Otherwise this can be an
-            integer or a RandomState class
-    Returns:
-        (numpy.random.RandomState) : rng -
-            a numpy random number generator
-
-    References:
-        .. [1] https://gitlab.kitware.com/computer-vision/kwarray/blob/master/kwarray/util_random.py#L270  # noqa: E501
-    """
-
-    if rng is None:
-        rng = np.random.mtrand._rand
-    elif isinstance(rng, int):
-        rng = np.random.RandomState(rng)
-    else:
-        rng = rng
-    return rng
-
-
-def random_boxes(num=1, scale=1, rng=None):
-    """Simple version of ``kwimage.Boxes.random``
-
-    Returns:
-        Tensor: shape (n, 4) in x1, y1, x2, y2 format.
-
-    References:
-        https://gitlab.kitware.com/computer-vision/kwimage/blob/master/kwimage/structs/boxes.py#L1390
-
-    Example:
-        >>> num = 3
-        >>> scale = 512
-        >>> rng = 0
-        >>> boxes = random_boxes(num, scale, rng)
-        >>> print(boxes)
-        tensor([[280.9925, 278.9802, 308.6148, 366.1769],
-                [216.9113, 330.6978, 224.0446, 456.5878],
-                [405.3632, 196.3221, 493.3953, 270.7942]])
-    """
-    rng = ensure_rng(rng)
-
-    tlbr = rng.rand(num, 4).astype(np.float32)
-
-    tl_x = np.minimum(tlbr[:, 0], tlbr[:, 2])
-    tl_y = np.minimum(tlbr[:, 1], tlbr[:, 3])
-    br_x = np.maximum(tlbr[:, 0], tlbr[:, 2])
-    br_y = np.maximum(tlbr[:, 1], tlbr[:, 3])
-
-    tlbr[:, 0] = tl_x * scale
-    tlbr[:, 1] = tl_y * scale
-    tlbr[:, 2] = br_x * scale
-    tlbr[:, 3] = br_y * scale
-
-    boxes = torch.from_numpy(tlbr)
-    return boxes
-
-
-class ModelEma(torch.nn.Module):
-    def __init__(self, model, decay=0.9997, device=None):
-        super(ModelEma, self).__init__()
-        # make a copy of the model for accumulating moving average of weights
-        self.module = deepcopy(model)
-        self.module.eval()
-
-        # import ipdb; ipdb.set_trace()
-
-        self.decay = decay
-        self.device = device  # perform ema on different device from model if set
-        if self.device is not None:
-            self.module.to(device=device)
-
-    def _update(self, model, update_fn):
-        with torch.no_grad():
-            for ema_v, model_v in zip(
-                self.module.state_dict().values(), model.state_dict().values()
-            ):
-                if self.device is not None:
-                    model_v = model_v.to(device=self.device)
-                ema_v.copy_(update_fn(ema_v, model_v))
-
-    def update(self, model):
-        self._update(model, update_fn=lambda e, m: self.decay * e + (1.0 - self.decay) * m)
-
-    def set(self, model):
-        self._update(model, update_fn=lambda e, m: m)
-
-
-class BestMetricSingle:
-    def __init__(self, init_res=0.0, better="large") -> None:
-        self.init_res = init_res
-        self.best_res = init_res
-        self.best_ep = -1
-
-        self.better = better
-        assert better in ["large", "small"]
-
-    def isbetter(self, new_res, old_res):
-        if self.better == "large":
-            return new_res > old_res
-        if self.better == "small":
-            return new_res < old_res
-
-    def update(self, new_res, ep):
-        if self.isbetter(new_res, self.best_res):
-            self.best_res = new_res
-            self.best_ep = ep
-            return True
-        return False
-
-    def __str__(self) -> str:
-        return "best_res: {}\t best_ep: {}".format(self.best_res, self.best_ep)
-
-    def __repr__(self) -> str:
-        return self.__str__()
-
-    def summary(self) -> dict:
-        return {
-            "best_res": self.best_res,
-            "best_ep": self.best_ep,
-        }
-
-
-class BestMetricHolder:
-    def __init__(self, init_res=0.0, better="large", use_ema=False) -> None:
-        self.best_all = BestMetricSingle(init_res, better)
-        self.use_ema = use_ema
-        if use_ema:
-            self.best_ema = BestMetricSingle(init_res, better)
-            self.best_regular = BestMetricSingle(init_res, better)
-
-    def update(self, new_res, epoch, is_ema=False):
-        """
-        return if the results is the best.
-        """
-        if not self.use_ema:
-            return self.best_all.update(new_res, epoch)
-        else:
-            if is_ema:
-                self.best_ema.update(new_res, epoch)
-                return self.best_all.update(new_res, epoch)
-            else:
-                self.best_regular.update(new_res, epoch)
-                return self.best_all.update(new_res, epoch)
-
-    def summary(self):
-        if not self.use_ema:
-            return self.best_all.summary()
-
-        res = {}
-        res.update({f"all_{k}": v for k, v in self.best_all.summary().items()})
-        res.update({f"regular_{k}": v for k, v in self.best_regular.summary().items()})
-        res.update({f"ema_{k}": v for k, v in self.best_ema.summary().items()})
-        return res
-
-    def __repr__(self) -> str:
-        return json.dumps(self.summary(), indent=2)
-
-    def __str__(self) -> str:
-        return self.__repr__()
-
-
-def targets_to(targets: List[Dict[str, Any]], device):
-    """Moves the target dicts to the given device."""
-    excluded_keys = [
-        "questionId",
-        "tokens_positive",
-        "strings_positive",
-        "tokens",
-        "dataset_name",
-        "sentence_id",
-        "original_img_id",
-        "nb_eval",
-        "task_id",
-        "original_id",
-        "token_span",
-        "caption",
-        "dataset_type",
-    ]
-    return [
-        {k: v.to(device) if k not in excluded_keys else v for k, v in t.items()} for t in targets
-    ]
-
-
-def get_phrases_from_posmap(
-    posmap: torch.BoolTensor, tokenized: Dict, tokenizer: AutoTokenizer, left_idx: int = 0, right_idx: int = 255
-):
-    assert isinstance(posmap, torch.Tensor), "posmap must be torch.Tensor"
-    if posmap.dim() == 1:
-        posmap[0: left_idx + 1] = False
-        posmap[right_idx:] = False
-        non_zero_idx = posmap.nonzero(as_tuple=True)[0].tolist()
-        token_ids = [tokenized["input_ids"][i] for i in non_zero_idx]
-        return tokenizer.decode(token_ids)
-    else:
-        raise NotImplementedError("posmap must be 1-dim")

grounding_dino/groundingdino/util/visualizer.py

@@ -1,318 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-@File    :   visualizer.py
-@Time    :   2022/04/05 11:39:33
-@Author  :   Shilong Liu 
-@Contact :   slongliu86@gmail.com
-"""
-
-import datetime
-import os
-
-import cv2
-import matplotlib.pyplot as plt
-import numpy as np
-import torch
-from matplotlib import transforms
-from matplotlib.collections import PatchCollection
-from matplotlib.patches import Polygon
-from pycocotools import mask as maskUtils
-
-
-def renorm(
-    img: torch.FloatTensor, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
-) -> torch.FloatTensor:
-    # img: tensor(3,H,W) or tensor(B,3,H,W)
-    # return: same as img
-    assert img.dim() == 3 or img.dim() == 4, "img.dim() should be 3 or 4 but %d" % img.dim()
-    if img.dim() == 3:
-        assert img.size(0) == 3, 'img.size(0) shoule be 3 but "%d". (%s)' % (
-            img.size(0),
-            str(img.size()),
-        )
-        img_perm = img.permute(1, 2, 0)
-        mean = torch.Tensor(mean)
-        std = torch.Tensor(std)
-        img_res = img_perm * std + mean
-        return img_res.permute(2, 0, 1)
-    else:  # img.dim() == 4
-        assert img.size(1) == 3, 'img.size(1) shoule be 3 but "%d". (%s)' % (
-            img.size(1),
-            str(img.size()),
-        )
-        img_perm = img.permute(0, 2, 3, 1)
-        mean = torch.Tensor(mean)
-        std = torch.Tensor(std)
-        img_res = img_perm * std + mean
-        return img_res.permute(0, 3, 1, 2)
-
-
-class ColorMap:
-    def __init__(self, basergb=[255, 255, 0]):
-        self.basergb = np.array(basergb)
-
-    def __call__(self, attnmap):
-        # attnmap: h, w. np.uint8.
-        # return: h, w, 4. np.uint8.
-        assert attnmap.dtype == np.uint8
-        h, w = attnmap.shape
-        res = self.basergb.copy()
-        res = res[None][None].repeat(h, 0).repeat(w, 1)  # h, w, 3
-        attn1 = attnmap.copy()[..., None]  # h, w, 1
-        res = np.concatenate((res, attn1), axis=-1).astype(np.uint8)
-        return res
-
-
-def rainbow_text(x, y, ls, lc, **kw):
-    """
-    Take a list of strings ``ls`` and colors ``lc`` and place them next to each
-    other, with text ls[i] being shown in color lc[i].
-
-    This example shows how to do both vertical and horizontal text, and will
-    pass all keyword arguments to plt.text, so you can set the font size,
-    family, etc.
-    """
-    t = plt.gca().transData
-    fig = plt.gcf()
-    plt.show()
-
-    # horizontal version
-    for s, c in zip(ls, lc):
-        text = plt.text(x, y, " " + s + " ", color=c, transform=t, **kw)
-        text.draw(fig.canvas.get_renderer())
-        ex = text.get_window_extent()
-        t = transforms.offset_copy(text._transform, x=ex.width, units="dots")
-
-    # #vertical version
-    # for s,c in zip(ls,lc):
-    #     text = plt.text(x,y," "+s+" ",color=c, transform=t,
-    #             rotation=90,va='bottom',ha='center',**kw)
-    #     text.draw(fig.canvas.get_renderer())
-    #     ex = text.get_window_extent()
-    #     t = transforms.offset_copy(text._transform, y=ex.height, units='dots')
-
-
-class COCOVisualizer:
-    def __init__(self, coco=None, tokenlizer=None) -> None:
-        self.coco = coco
-
-    def visualize(self, img, tgt, caption=None, dpi=180, savedir="vis"):
-        """
-        img: tensor(3, H, W)
-        tgt: make sure they are all on cpu.
-            must have items: 'image_id', 'boxes', 'size'
-        """
-        plt.figure(dpi=dpi)
-        plt.rcParams["font.size"] = "5"
-        ax = plt.gca()
-        img = renorm(img).permute(1, 2, 0)
-        # if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO':
-        #     import ipdb; ipdb.set_trace()
-        ax.imshow(img)
-
-        self.addtgt(tgt)
-
-        if tgt is None:
-            image_id = 0
-        elif "image_id" not in tgt:
-            image_id = 0
-        else:
-            image_id = tgt["image_id"]
-
-        if caption is None:
-            savename = "{}/{}-{}.png".format(
-                savedir, int(image_id), str(datetime.datetime.now()).replace(" ", "-")
-            )
-        else:
-            savename = "{}/{}-{}-{}.png".format(
-                savedir, caption, int(image_id), str(datetime.datetime.now()).replace(" ", "-")
-            )
-        print("savename: {}".format(savename))
-        os.makedirs(os.path.dirname(savename), exist_ok=True)
-        plt.savefig(savename)
-        plt.close()
-
-    def addtgt(self, tgt):
-        """ """
-        if tgt is None or not "boxes" in tgt:
-            ax = plt.gca()
-
-            if "caption" in tgt:
-                ax.set_title(tgt["caption"], wrap=True)
-
-            ax.set_axis_off()
-            return
-
-        ax = plt.gca()
-        H, W = tgt["size"]
-        numbox = tgt["boxes"].shape[0]
-
-        color = []
-        polygons = []
-        boxes = []
-        for box in tgt["boxes"].cpu():
-            unnormbbox = box * torch.Tensor([W, H, W, H])
-            unnormbbox[:2] -= unnormbbox[2:] / 2
-            [bbox_x, bbox_y, bbox_w, bbox_h] = unnormbbox.tolist()
-            boxes.append([bbox_x, bbox_y, bbox_w, bbox_h])
-            poly = [
-                [bbox_x, bbox_y],
-                [bbox_x, bbox_y + bbox_h],
-                [bbox_x + bbox_w, bbox_y + bbox_h],
-                [bbox_x + bbox_w, bbox_y],
-            ]
-            np_poly = np.array(poly).reshape((4, 2))
-            polygons.append(Polygon(np_poly))
-            c = (np.random.random((1, 3)) * 0.6 + 0.4).tolist()[0]
-            color.append(c)
-
-        p = PatchCollection(polygons, facecolor=color, linewidths=0, alpha=0.1)
-        ax.add_collection(p)
-        p = PatchCollection(polygons, facecolor="none", edgecolors=color, linewidths=2)
-        ax.add_collection(p)
-
-        if "strings_positive" in tgt and len(tgt["strings_positive"]) > 0:
-            assert (
-                len(tgt["strings_positive"]) == numbox
-            ), f"{len(tgt['strings_positive'])} = {numbox}, "
-            for idx, strlist in enumerate(tgt["strings_positive"]):
-                cate_id = int(tgt["labels"][idx])
-                _string = str(cate_id) + ":" + " ".join(strlist)
-                bbox_x, bbox_y, bbox_w, bbox_h = boxes[idx]
-                # ax.text(bbox_x, bbox_y, _string, color='black', bbox={'facecolor': 'yellow', 'alpha': 1.0, 'pad': 1})
-                ax.text(
-                    bbox_x,
-                    bbox_y,
-                    _string,
-                    color="black",
-                    bbox={"facecolor": color[idx], "alpha": 0.6, "pad": 1},
-                )
-
-        if "box_label" in tgt:
-            assert len(tgt["box_label"]) == numbox, f"{len(tgt['box_label'])} = {numbox}, "
-            for idx, bl in enumerate(tgt["box_label"]):
-                _string = str(bl)
-                bbox_x, bbox_y, bbox_w, bbox_h = boxes[idx]
-                # ax.text(bbox_x, bbox_y, _string, color='black', bbox={'facecolor': 'yellow', 'alpha': 1.0, 'pad': 1})
-                ax.text(
-                    bbox_x,
-                    bbox_y,
-                    _string,
-                    color="black",
-                    bbox={"facecolor": color[idx], "alpha": 0.6, "pad": 1},
-                )
-
-        if "caption" in tgt:
-            ax.set_title(tgt["caption"], wrap=True)
-            # plt.figure()
-            # rainbow_text(0.0,0.0,"all unicorns poop rainbows ! ! !".split(),
-            #         ['red', 'orange', 'brown', 'green', 'blue', 'purple', 'black'])
-
-        if "attn" in tgt:
-            # if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO':
-            #     import ipdb; ipdb.set_trace()
-            if isinstance(tgt["attn"], tuple):
-                tgt["attn"] = [tgt["attn"]]
-            for item in tgt["attn"]:
-                attn_map, basergb = item
-                attn_map = (attn_map - attn_map.min()) / (attn_map.max() - attn_map.min() + 1e-3)
-                attn_map = (attn_map * 255).astype(np.uint8)
-                cm = ColorMap(basergb)
-                heatmap = cm(attn_map)
-                ax.imshow(heatmap)
-        ax.set_axis_off()
-
-    def showAnns(self, anns, draw_bbox=False):
-        """
-        Display the specified annotations.
-        :param anns (array of object): annotations to display
-        :return: None
-        """
-        if len(anns) == 0:
-            return 0
-        if "segmentation" in anns[0] or "keypoints" in anns[0]:
-            datasetType = "instances"
-        elif "caption" in anns[0]:
-            datasetType = "captions"
-        else:
-            raise Exception("datasetType not supported")
-        if datasetType == "instances":
-            ax = plt.gca()
-            ax.set_autoscale_on(False)
-            polygons = []
-            color = []
-            for ann in anns:
-                c = (np.random.random((1, 3)) * 0.6 + 0.4).tolist()[0]
-                if "segmentation" in ann:
-                    if type(ann["segmentation"]) == list:
-                        # polygon
-                        for seg in ann["segmentation"]:
-                            poly = np.array(seg).reshape((int(len(seg) / 2), 2))
-                            polygons.append(Polygon(poly))
-                            color.append(c)
-                    else:
-                        # mask
-                        t = self.imgs[ann["image_id"]]
-                        if type(ann["segmentation"]["counts"]) == list:
-                            rle = maskUtils.frPyObjects(
-                                [ann["segmentation"]], t["height"], t["width"]
-                            )
-                        else:
-                            rle = [ann["segmentation"]]
-                        m = maskUtils.decode(rle)
-                        img = np.ones((m.shape[0], m.shape[1], 3))
-                        if ann["iscrowd"] == 1:
-                            color_mask = np.array([2.0, 166.0, 101.0]) / 255
-                        if ann["iscrowd"] == 0:
-                            color_mask = np.random.random((1, 3)).tolist()[0]
-                        for i in range(3):
-                            img[:, :, i] = color_mask[i]
-                        ax.imshow(np.dstack((img, m * 0.5)))
-                if "keypoints" in ann and type(ann["keypoints"]) == list:
-                    # turn skeleton into zero-based index
-                    sks = np.array(self.loadCats(ann["category_id"])[0]["skeleton"]) - 1
-                    kp = np.array(ann["keypoints"])
-                    x = kp[0::3]
-                    y = kp[1::3]
-                    v = kp[2::3]
-                    for sk in sks:
-                        if np.all(v[sk] > 0):
-                            plt.plot(x[sk], y[sk], linewidth=3, color=c)
-                    plt.plot(
-                        x[v > 0],
-                        y[v > 0],
-                        "o",
-                        markersize=8,
-                        markerfacecolor=c,
-                        markeredgecolor="k",
-                        markeredgewidth=2,
-                    )
-                    plt.plot(
-                        x[v > 1],
-                        y[v > 1],
-                        "o",
-                        markersize=8,
-                        markerfacecolor=c,
-                        markeredgecolor=c,
-                        markeredgewidth=2,
-                    )
-
-                if draw_bbox:
-                    [bbox_x, bbox_y, bbox_w, bbox_h] = ann["bbox"]
-                    poly = [
-                        [bbox_x, bbox_y],
-                        [bbox_x, bbox_y + bbox_h],
-                        [bbox_x + bbox_w, bbox_y + bbox_h],
-                        [bbox_x + bbox_w, bbox_y],
-                    ]
-                    np_poly = np.array(poly).reshape((4, 2))
-                    polygons.append(Polygon(np_poly))
-                    color.append(c)
-
-            # p = PatchCollection(polygons, facecolor=color, linewidths=0, alpha=0.4)
-            # ax.add_collection(p)
-            p = PatchCollection(polygons, facecolor="none", edgecolors=color, linewidths=2)
-            ax.add_collection(p)
-        elif datasetType == "captions":
-            for ann in anns:
-                print(ann["caption"])

grounding_dino/groundingdino/util/vl_utils.py

@@ -1,100 +0,0 @@
-import os
-import random
-from typing import List
-
-import torch
-
-
-def create_positive_map_from_span(tokenized, token_span, max_text_len=256):
-    """construct a map such that positive_map[i,j] = True iff box i is associated to token j
-    Input:
-        - tokenized:
-            - input_ids: Tensor[1, ntokens]
-            - attention_mask: Tensor[1, ntokens]
-        - token_span: list with length num_boxes.
-            - each item: [start_idx, end_idx]
-    """
-    positive_map = torch.zeros((len(token_span), max_text_len), dtype=torch.float)
-    for j, tok_list in enumerate(token_span):
-        for (beg, end) in tok_list:
-            beg_pos = tokenized.char_to_token(beg)
-            end_pos = tokenized.char_to_token(end - 1)
-            if beg_pos is None:
-                try:
-                    beg_pos = tokenized.char_to_token(beg + 1)
-                    if beg_pos is None:
-                        beg_pos = tokenized.char_to_token(beg + 2)
-                except:
-                    beg_pos = None
-            if end_pos is None:
-                try:
-                    end_pos = tokenized.char_to_token(end - 2)
-                    if end_pos is None:
-                        end_pos = tokenized.char_to_token(end - 3)
-                except:
-                    end_pos = None
-            if beg_pos is None or end_pos is None:
-                continue
-
-            assert beg_pos is not None and end_pos is not None
-            if os.environ.get("SHILONG_DEBUG_ONLY_ONE_POS", None) == "TRUE":
-                positive_map[j, beg_pos] = 1
-                break
-            else:
-                positive_map[j, beg_pos : end_pos + 1].fill_(1)
-
-    return positive_map / (positive_map.sum(-1)[:, None] + 1e-6)
-
-
-def build_captions_and_token_span(cat_list, force_lowercase):
-    """
-    Return:
-        captions: str
-        cat2tokenspan: dict
-            {
-                'dog': [[0, 2]],
-                ...
-            }
-    """
-
-    cat2tokenspan = {}
-    captions = ""
-    for catname in cat_list:
-        class_name = catname
-        if force_lowercase:
-            class_name = class_name.lower()
-        if "/" in class_name:
-            class_name_list: List = class_name.strip().split("/")
-            class_name_list.append(class_name)
-            class_name: str = random.choice(class_name_list)
-
-        tokens_positive_i = []
-        subnamelist = [i.strip() for i in class_name.strip().split(" ")]
-        for subname in subnamelist:
-            if len(subname) == 0:
-                continue
-            if len(captions) > 0:
-                captions = captions + " "
-            strat_idx = len(captions)
-            end_idx = strat_idx + len(subname)
-            tokens_positive_i.append([strat_idx, end_idx])
-            captions = captions + subname
-
-        if len(tokens_positive_i) > 0:
-            captions = captions + " ."
-            cat2tokenspan[class_name] = tokens_positive_i
-
-    return captions, cat2tokenspan
-
-
-def build_id2posspan_and_caption(category_dict: dict):
-    """Build id2pos_span and caption from category_dict
-
-    Args:
-        category_dict (dict): category_dict
-    """
-    cat_list = [item["name"].lower() for item in category_dict]
-    id2catname = {item["id"]: item["name"].lower() for item in category_dict}
-    caption, cat2posspan = build_captions_and_token_span(cat_list, force_lowercase=True)
-    id2posspan = {catid: cat2posspan[catname] for catid, catname in id2catname.items()}
-    return id2posspan, caption

grounding_dino/groundingdino/version.py

@@ -1 +0,0 @@
-__version__ = '0.1.0'

grounding_dino/requirements.txt

@@ -1,10 +0,0 @@
-torch
-torchvision
-transformers
-addict
-yapf
-timm
-numpy
-opencv-python
-supervision>=0.22.0
-pycocotools

grounding_dino/setup.py

@@ -1,224 +0,0 @@
-# coding=utf-8
-# Copyright 2022 The IDEA Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ------------------------------------------------------------------------------------------------
-# Modified from
-# https://github.com/fundamentalvision/Deformable-DETR/blob/main/models/ops/setup.py
-# https://github.com/facebookresearch/detectron2/blob/main/setup.py
-# https://github.com/open-mmlab/mmdetection/blob/master/setup.py
-# https://github.com/Oneflow-Inc/libai/blob/main/setup.py
-# ------------------------------------------------------------------------------------------------
-
-import glob
-import os
-import subprocess
-
-import subprocess
-import sys
-
-def install_torch():
-    try:
-        import torch
-    except ImportError:
-        subprocess.check_call([sys.executable, "-m", "pip", "install", "torch"])
-
-# Call the function to ensure torch is installed
-install_torch()
-
-import torch
-from setuptools import find_packages, setup
-from torch.utils.cpp_extension import CUDA_HOME, CppExtension, CUDAExtension
-
-# groundingdino version info
-version = "0.1.0"
-package_name = "groundingdino"
-cwd = os.path.dirname(os.path.abspath(__file__))
-
-
-sha = "Unknown"
-try:
-    sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=cwd).decode("ascii").strip()
-except Exception:
-    pass
-
-
-def write_version_file():
-    version_path = os.path.join(cwd, "groundingdino", "version.py")
-    with open(version_path, "w") as f:
-        f.write(f"__version__ = '{version}'\n")
-        # f.write(f"git_version = {repr(sha)}\n")
-
-
-requirements = ["torch", "torchvision"]
-
-torch_ver = [int(x) for x in torch.__version__.split(".")[:2]]
-
-
-def get_extensions():
-    this_dir = os.path.dirname(os.path.abspath(__file__))
-    extensions_dir = os.path.join(this_dir, "groundingdino", "models", "GroundingDINO", "csrc")
-
-    main_source = os.path.join(extensions_dir, "vision.cpp")
-    sources = glob.glob(os.path.join(extensions_dir, "**", "*.cpp"))
-    source_cuda = glob.glob(os.path.join(extensions_dir, "**", "*.cu")) + glob.glob(
-        os.path.join(extensions_dir, "*.cu")
-    )
-
-    sources = [main_source] + sources
-
-    extension = CppExtension
-
-    extra_compile_args = {"cxx": []}
-    define_macros = []
-
-    if CUDA_HOME is not None and (torch.cuda.is_available() or "TORCH_CUDA_ARCH_LIST" in os.environ):
-        print("Compiling with CUDA")
-        extension = CUDAExtension
-        sources += source_cuda
-        define_macros += [("WITH_CUDA", None)]
-        extra_compile_args["nvcc"] = [
-            "-DCUDA_HAS_FP16=1",
-            "-D__CUDA_NO_HALF_OPERATORS__",
-            "-D__CUDA_NO_HALF_CONVERSIONS__",
-            "-D__CUDA_NO_HALF2_OPERATORS__",
-            "-gencode=arch=compute_70,code=sm_70",
-            "-gencode=arch=compute_75,code=sm_75",
-            "-gencode=arch=compute_80,code=sm_80",
-            "-gencode=arch=compute_86,code=sm_86",
-        ]
-    else:
-        print("Compiling without CUDA")
-        define_macros += [("WITH_HIP", None)]
-        extra_compile_args["nvcc"] = []
-        return None
-
-    sources = [os.path.join(extensions_dir, s) for s in sources]
-    include_dirs = [extensions_dir]
-
-    ext_modules = [
-        extension(
-            "groundingdino._C",
-            sources,
-            include_dirs=include_dirs,
-            define_macros=define_macros,
-            extra_compile_args=extra_compile_args,
-        )
-    ]
-
-    return ext_modules
-
-
-def parse_requirements(fname="requirements.txt", with_version=True):
-    """Parse the package dependencies listed in a requirements file but strips
-    specific versioning information.
-
-    Args:
-        fname (str): path to requirements file
-        with_version (bool, default=False): if True include version specs
-
-    Returns:
-        List[str]: list of requirements items
-
-    CommandLine:
-        python -c "import setup; print(setup.parse_requirements())"
-    """
-    import re
-    import sys
-    from os.path import exists
-
-    require_fpath = fname
-
-    def parse_line(line):
-        """Parse information from a line in a requirements text file."""
-        if line.startswith("-r "):
-            # Allow specifying requirements in other files
-            target = line.split(" ")[1]
-            for info in parse_require_file(target):
-                yield info
-        else:
-            info = {"line": line}
-            if line.startswith("-e "):
-                info["package"] = line.split("#egg=")[1]
-            elif "@git+" in line:
-                info["package"] = line
-            else:
-                # Remove versioning from the package
-                pat = "(" + "|".join([">=", "==", ">"]) + ")"
-                parts = re.split(pat, line, maxsplit=1)
-                parts = [p.strip() for p in parts]
-
-                info["package"] = parts[0]
-                if len(parts) > 1:
-                    op, rest = parts[1:]
-                    if ";" in rest:
-                        # Handle platform specific dependencies
-                        # http://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-platform-specific-dependencies
-                        version, platform_deps = map(str.strip, rest.split(";"))
-                        info["platform_deps"] = platform_deps
-                    else:
-                        version = rest  # NOQA
-                    info["version"] = (op, version)
-            yield info
-
-    def parse_require_file(fpath):
-        with open(fpath, "r") as f:
-            for line in f.readlines():
-                line = line.strip()
-                if line and not line.startswith("#"):
-                    for info in parse_line(line):
-                        yield info
-
-    def gen_packages_items():
-        if exists(require_fpath):
-            for info in parse_require_file(require_fpath):
-                parts = [info["package"]]
-                if with_version and "version" in info:
-                    parts.extend(info["version"])
-                if not sys.version.startswith("3.4"):
-                    # apparently package_deps are broken in 3.4
-                    platform_deps = info.get("platform_deps")
-                    if platform_deps is not None:
-                        parts.append(";" + platform_deps)
-                item = "".join(parts)
-                yield item
-
-    packages = list(gen_packages_items())
-    return packages
-
-
-if __name__ == "__main__":
-    print(f"Building wheel {package_name}-{version}")
-
-    with open("LICENSE", "r", encoding="utf-8") as f:
-        license = f.read()
-
-    write_version_file()
-
-    setup(
-        name="groundingdino",
-        version="0.1.0",
-        author="International Digital Economy Academy, Shilong Liu",
-        url="https://github.com/IDEA-Research/GroundingDINO",
-        description="open-set object detector",
-        license=license,
-        # install_requires=parse_requirements("requirements.txt"),
-        packages=find_packages(
-            exclude=(
-                "configs",
-                "tests",
-            )
-        ),
-        ext_modules=get_extensions(),
-        cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension},
-    )

grounding_dino/test.ipynb

@@ -1,114 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "final text_encoder_type: bert-base-uncased\n"
-     ]
-    },
-    {
-     "data": {
-      "application/json": {
-       "ascii": false,
-       "bar_format": null,
-       "colour": null,
-       "elapsed": 0.014210224151611328,
-       "initial": 0,
-       "n": 0,
-       "ncols": null,
-       "nrows": null,
-       "postfix": null,
-       "prefix": "Downloading model.safetensors",
-       "rate": null,
-       "total": 440449768,
-       "unit": "B",
-       "unit_divisor": 1000,
-       "unit_scale": true
-      },
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "5922f34578364d36afa13de9f01254bd",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading model.safetensors:   0%|          | 0.00/440M [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/root/miniconda3/lib/python3.8/site-packages/transformers/modeling_utils.py:881: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers.\n",
-      "  warnings.warn(\n",
-      "/root/miniconda3/lib/python3.8/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
-      "  warnings.warn(\"None of the inputs have requires_grad=True. Gradients will be None\")\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "True"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from groundingdino.util.inference import load_model, load_image, predict, annotate\n",
-    "import cv2\n",
-    "\n",
-    "model = load_model(\"groundingdino/config/GroundingDINO_SwinT_OGC.py\", \"../04-06-segment-anything/weights/groundingdino_swint_ogc.pth\")\n",
-    "IMAGE_PATH = \".asset/cat_dog.jpeg\"\n",
-    "TEXT_PROMPT = \"chair . person . dog .\"\n",
-    "BOX_TRESHOLD = 0.35\n",
-    "TEXT_TRESHOLD = 0.25\n",
-    "\n",
-    "image_source, image = load_image(IMAGE_PATH)\n",
-    "\n",
-    "boxes, logits, phrases = predict(\n",
-    "    model=model,\n",
-    "    image=image,\n",
-    "    caption=TEXT_PROMPT,\n",
-    "    box_threshold=BOX_TRESHOLD,\n",
-    "    text_threshold=TEXT_TRESHOLD\n",
-    ")\n",
-    "\n",
-    "annotated_frame = annotate(image_source=image_source, boxes=boxes, logits=logits, phrases=phrases)\n",
-    "cv2.imwrite(\"annotated_image.jpg\", annotated_frame)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "base",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.10"
-  },
-  "orig_nbformat": 4
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}