add readme (#10)

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* remove submodule

* add mPLUG MiniGPT4

* Update Readme.md

* Update Readme.md

* Update Readme.md

---------

Co-authored-by: Yuliang Liu <34134635+Yuliang-Liu@users.noreply.github.com>

models/mPLUG_Owl/mplug_owl/__init__.py

@@ -0,0 +1,77 @@
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from transformers.utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available
+
+
+_import_structure = {
+    "configuration_mplug_owl": ["MPLUG_OWL_PRETRAINED_CONFIG_ARCHIVE_MAP", "MplugOwlConfig"],
+    "processing_mplug_owl": ["MplugOwlImageProcessor", "MplugOwlProcessor"],
+    "tokenization_mplug_owl": ["MplugOwlTokenizer"],
+}
+
+try:
+    if not is_tokenizers_available():
+        raise OptionalDependencyNotAvailable()
+except OptionalDependencyNotAvailable:
+    pass
+
+
+try:
+    if not is_torch_available():
+        raise OptionalDependencyNotAvailable()
+except OptionalDependencyNotAvailable:
+    pass
+else:
+    _import_structure["modeling_mplug_owl"] = [
+        "MPLUG_OWL_PRETRAINED_MODEL_ARCHIVE_LIST",
+        "MplugOwlForConditionalGeneration",
+        "MplugOwlModel",
+    ]
+
+
+if TYPE_CHECKING:
+    from .configuration_mplug_owl import MPLUG_OWL_PRETRAINED_CONFIG_ARCHIVE_MAP, MplugOwlConfig
+    from .tokenization_mplug_owl import MplugOwlTokenizer
+
+    try:
+        if not is_tokenizers_available():
+            raise OptionalDependencyNotAvailable()
+    except OptionalDependencyNotAvailable:
+        pass
+
+    try:
+        if not is_torch_available():
+            raise OptionalDependencyNotAvailable()
+    except OptionalDependencyNotAvailable:
+        pass
+    else:
+        from .modeling_mplug_owl import (
+            MPLUG_OWL_PRETRAINED_MODEL_ARCHIVE_LIST,
+            MplugOwlForConditionalGeneration,
+            MplugOwlModel,
+            MplugOwlPreTrainedModel,
+        )
+
+
+else:
+    import sys
+
+    sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
+
+from .configuration_mplug_owl import *
+from .modeling_mplug_owl import *
+from .processing_mplug_owl import *
+from .tokenization_mplug_owl import *

models/mPLUG_Owl/mplug_owl/configuration_mplug_owl.py

@@ -0,0 +1,298 @@
+# coding=utf-8
+# Copyright 2022 x-plug and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" MplugOwl model configuration"""
+import copy
+import os
+from typing import Union
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
+from transformers.utils import logging
+from transformers.models.auto import CONFIG_MAPPING
+
+
+logger = logging.get_logger(__name__)
+
+MPLUG_OWL_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    "MAGAer13/mplug-owl-llama-7b": "https://huggingface.co/MAGAer13/mplug-owl-llama-7b/resolve/main/config.json",
+    # See all MplugOwl models at https://huggingface.co/models?filter=mplug_owl
+}
+
+
+class MplugOwlVisionConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`MplugOwlVisionModel`]. It is used to instantiate
+    a
+     mPLUG-Owl vision encoder according to the specified arguments, defining the model architecture. Instantiating a
+     configuration defaults will yield a similar configuration to that of the mPLUG-Owl
+     [x-plug/x_plug-llama-7b](https://huggingface.co/x-plug/x_plug-llama-7b) architecture.
+
+     Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+     documentation from [`PretrainedConfig`] for more information.
+
+     Args:
+         hidden_size (`int`, *optional*, defaults to 768):
+             Dimensionality of the encoder layers and the pooler layer.
+         intermediate_size (`int`, *optional*, defaults to 3072):
+             Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+         num_hidden_layers (`int`, *optional*, defaults to 12):
+             Number of hidden layers in the Transformer encoder.
+         num_attention_heads (`int`, *optional*, defaults to 12):
+             Number of attention heads for each attention layer in the Transformer encoder.
+         image_size (`int`, *optional*, defaults to 224):
+             The size (resolution) of each image.
+         patch_size (`int`, *optional*, defaults to 32):
+             The size (resolution) of each patch.
+         hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):
+             The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+             `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
+         layer_norm_eps (`float`, *optional*, defaults to 1e-5):
+             The epsilon used by the layer normalization layers.
+         attention_dropout (`float`, *optional*, defaults to 0.0):
+             The dropout ratio for the attention probabilities.
+         initializer_range (`float`, *optional*, defaults to 0.02):
+             The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+         initializer_factor (`float`, *optional*, defaults to 1):
+             A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
+             testing).
+
+
+     ```"""
+
+    model_type = "mplug_owl_vision_model"
+
+    def __init__(
+        self,
+        hidden_size=1024,
+        intermediate_size=4096,
+        projection_dim=768,
+        num_hidden_layers=24,
+        num_attention_heads=16,
+        num_channels=3,
+        image_size=224,
+        patch_size=14,
+        hidden_act="quick_gelu",
+        layer_norm_eps=1e-6,
+        attention_dropout=0.0,
+        initializer_range=0.02,
+        initializer_factor=1.0,
+        use_flash_attn=False,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.projection_dim = projection_dim
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_channels = num_channels
+        self.patch_size = patch_size
+        self.image_size = image_size
+        self.initializer_range = initializer_range
+        self.initializer_factor = initializer_factor
+        self.attention_dropout = attention_dropout
+        self.layer_norm_eps = layer_norm_eps
+        self.hidden_act = hidden_act
+        self.use_flash_attn = use_flash_attn
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
+        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+        # get the vision config dict if we are loading from MplugOwlConfig
+        if config_dict.get("model_type") == "mplug-owl":
+            config_dict = config_dict["vision_config"]
+
+        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+            logger.warning(
+                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+            )
+
+        return cls.from_dict(config_dict, **kwargs)
+
+
+class MplugOwlVisualAbstractorConfig(PretrainedConfig):
+    model_type = "mplug_owl_visual_abstract"
+
+    def __init__(
+        self,
+        hidden_size=1024,  #
+        num_hidden_layers=6,  #
+        num_attention_heads=16,  #
+        intermediate_size=4096,  #
+        attention_probs_dropout_prob=0.1,  #
+        initializer_range=0.02,
+        layer_norm_eps=1e-6,  #
+        encoder_hidden_size=1024,  #
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.intermediate_size = intermediate_size
+        self.attention_probs_dropout_prob = attention_probs_dropout_prob
+        self.initializer_range = initializer_range
+        self.layer_norm_eps = layer_norm_eps
+        self.encoder_hidden_size = encoder_hidden_size
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
+        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+        # get the visual_abstractor config dict if we are loading from MplugOwlConfig
+        if config_dict.get("model_type") == "mplug-owl":
+            config_dict = config_dict["abstractor_config"]
+
+        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+            logger.warning(
+                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+            )
+
+        return cls.from_dict(config_dict, **kwargs)
+
+
+class MplugOwlConfig(PretrainedConfig):
+    r"""
+    [`MplugOwlConfig`] is the configuration class to store the configuration of a [`MplugOwlForConditionalGeneration`].
+    It is used to instantiate a mPLUG-Owl model according to the specified arguments, defining the vision model,
+    Q-Former model and language model configs. Instantiating a configuration with the defaults will yield a similar
+    configuration to that of the mPLUG-Owl [x-plug/x_plug-llama-7b](https://huggingface.co/x-plug/x_plug-llama-7b)
+    architecture.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        vision_config (`dict`, *optional*):
+            Dictionary of configuration options used to initialize [`MplugOwlVisionConfig`].
+        visual_abstractor_config (`dict`, *optional*):
+            Dictionary of configuration options used to initialize [`MplugOwlVisualAbstractorConfig`].
+        text_config (`dict`, *optional*):
+            Dictionary of configuration options used to initialize any [`PretrainedConfig`].
+        num_query_tokens (`int`, *optional*, defaults to 32):
+            The number of query tokens passed through the Transformer.
+
+        kwargs (*optional*):
+            Dictionary of keyword arguments.
+
+    Example:
+
+    ```python
+    >>> from transformers import (
+    ...     MplugOwlVisionConfig,
+    ...     MplugOwlVisualAbstractorConfig,
+    ...     OPTConfig,
+    ...     MplugOwlConfig,
+    ...     MplugOwlForConditionalGeneration,
+    ... )
+
+    >>> # Initializing a MplugOwlConfig with x-plug/x_plug-llama-7b style configuration
+    >>> configuration = MplugOwlConfig()
+
+    >>> # Initializing a MplugOwlForConditionalGeneration (with random weights) from the x-plug/x_plug-llama-7b style configuration
+    >>> model = MplugOwlForConditionalGeneration(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+
+    >>> # We can also initialize a MplugOwlConfig from a MplugOwlVisionConfig, MplugOwlVisualAbstractorConfig and any PretrainedConfig
+
+    >>> # Initializing mPLUG-Owl vision, mPLUG-Owl Q-Former and language model configurations
+    >>> vision_config = MplugOwlVisionConfig()
+    >>> visual_abstractor_config = MplugOwlVisualAbstractorConfig()
+    >>> text_config = OPTConfig()
+
+    >>> config = MplugOwlConfig.from_text_vision_configs(vision_config, visual_abstractor_config, text_config)
+    ```"""
+    model_type = "mplug-owl"
+    is_composition = True
+
+    def __init__(
+        self, vision_config=None, visual_abstractor_config=None, text_config=None, num_query_tokens=64, **kwargs
+    ):
+        super().__init__(**kwargs)
+        if vision_config is None:
+            vision_config = MplugOwlVisionConfig().to_dict()
+            logger.info("vision_config is None.")
+
+        if visual_abstractor_config is None:
+            visual_abstractor_config = {}
+            logger.info("abstractor_config is None. ")
+
+        if text_config is None:
+            # we use LLAMA 7b by default
+            from transformers.llama.configuration_llama import LlamaConfig
+
+            text_config = LlamaConfig(pad_token_id=2).to_dict()
+            logger.info("text_config is None.")
+
+        self.vision_config = MplugOwlVisionConfig(**vision_config)
+        self.visual_abstractor_config = MplugOwlVisualAbstractorConfig(**visual_abstractor_config)
+        # self.visual_abstractor_config.layer_norm_eps = 1e-6
+        text_model_type = text_config["model_type"] if "model_type" in text_config else "llama"
+        self.text_config = CONFIG_MAPPING[text_model_type](**text_config)
+
+        self.tie_word_embeddings = self.text_config.tie_word_embeddings
+        self.is_encoder_decoder = self.text_config.is_encoder_decoder
+
+        self.num_query_tokens = num_query_tokens
+        # self.visual_abstractor_config.encoder_hidden_size = self.vision_config.hidden_size
+        self.use_decoder_only_language_model = self.text_config.model_type in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
+        self.initializer_factor = 1.0
+        self.initializer_range = 0.02
+
+        for attr in dir(self.text_config):
+            if not hasattr(self, attr):
+                setattr(self, attr, getattr(self.text_config, attr))
+
+    @classmethod
+    def from_vision_visual_abstractor_text_configs(
+        cls,
+        vision_config: MplugOwlVisionConfig,
+        visual_abstractor_config: MplugOwlVisualAbstractorConfig,
+        text_config: PretrainedConfig,
+        **kwargs,
+    ):
+        r"""
+        Instantiate a [`MplugOwlConfig`] (or a derived class) from a mPLUG-Owl vision model, Q-Former and language
+        model configurations.
+
+        Returns:
+            [`MplugOwlConfig`]: An instance of a configuration object
+        """
+
+        return cls(
+            vision_config=vision_config.to_dict(),
+            visual_abstractor_config=visual_abstractor_config.to_dict(),
+            text_config=text_config.to_dict(),
+            **kwargs,
+        )
+
+    def to_dict(self):
+        """
+        Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
+
+        Returns:
+            `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
+        """
+        output = copy.deepcopy(self.__dict__)
+        output["vision_config"] = self.vision_config.to_dict()
+        output["visual_abstractor_config"] = self.visual_abstractor_config.to_dict()
+        output["text_config"] = self.text_config.to_dict()
+        output["model_type"] = self.__class__.model_type
+        return output

models/mPLUG_Owl/mplug_owl/processing_mplug_owl.py

@@ -0,0 +1,208 @@
+import re
+
+import torch
+import torch.utils.checkpoint
+
+from transformers.processing_utils import ProcessorMixin
+from transformers.tokenization_utils_base import BatchEncoding
+from transformers.models.clip.image_processing_clip import CLIPImageProcessor
+
+
+media_token = ({"image": ("<image>", 65)},)
+
+
+class MplugOwlProcessor(ProcessorMixin):
+    attributes = []
+    tokenizer_class = ("MplugOwlTokenizer")
+
+    def __init__(self, image_processor=None, tokenizer=None, **kwargs):
+        super().__init__(**kwargs)
+        self.tokens_to_generate = 0
+        self.image_processor = image_processor
+        self.tokenizer = tokenizer
+        self.add_BOS = True
+
+    def __call__(self, text=None, images=None, return_tensors=None, **kwargs):
+        if text is None and images is None:
+            raise ValueError("You have to specify either text or images. Both cannot be none.")
+
+        if text is not None:
+            encoding = tokenize_prompts(
+                prompts=text,
+                tokens_to_generate=self.tokens_to_generate,
+                add_BOS=self.add_BOS,
+                tokenizer=self.tokenizer,
+                ignore_dist=True,
+                **kwargs,
+            )
+            # encoding = self.tokenizer(text, return_tensors=return_tensors, **kwargs)
+
+        if images is not None:
+            image_features = self.image_processor(images, return_tensors=return_tensors, **kwargs)
+
+        if text is not None and images is not None:
+            encoding["pixel_values"] = image_features.pixel_values
+            return BatchEncoding(data=encoding)
+        elif text is not None:
+            return BatchEncoding(data=encoding)
+        else:
+            return BatchEncoding(data=dict(**image_features), tensor_type=return_tensors)
+
+    def batch_decode(self, skip_special_tokens=True, *args, **kwargs):
+        """
+        This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
+        refer to the docstring of this method for more information.
+        """
+        return self.tokenizer.batch_decode(*args, skip_special_tokens=skip_special_tokens, **kwargs)
+
+    def decode(self, skip_special_tokens=True, *args, **kwargs):
+        """
+        This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
+        the docstring of this method for more information.
+        """
+        return self.tokenizer.decode(*args, skip_special_tokens=skip_special_tokens, **kwargs)
+
+
+class MplugOwlImageProcessor(CLIPImageProcessor):
+    pass
+
+
+def detokenize_generations(tokens_gpu_tensor, lengths_gpu_tensor, return_segments, tokenizer):
+    """Detokenize the generated tokens."""
+
+    prompts_plus_generations = []
+    if return_segments:
+        prompts_plus_generations_segments = []
+
+    tokens = tokens_gpu_tensor.cpu().numpy().tolist()
+    lengths = lengths_gpu_tensor.cpu().numpy().tolist()
+    for sequence_tokens, length in zip(tokens, lengths):
+        sequence_tokens = sequence_tokens[:length]
+        prompts_plus_generations.append(tokenizer.detokenize(sequence_tokens))
+        if return_segments:
+            from tokenizers.decoders import Metaspace
+
+            if hasattr(tokenizer, "tokenizer"):
+                if isinstance(tokenizer.tokenizer.decoder, Metaspace):
+                    words = tokenizer.tokenizer.decode(sequence_tokens)
+                else:
+                    words = []
+                    for token in sequence_tokens:
+                        word = tokenizer.tokenizer.decoder[token]
+                        word = bytearray([tokenizer.tokenizer.byte_decoder[c] for c in word]).decode(
+                            "utf-8", errors="replace"
+                        )
+                        words.append(word)
+                prompts_plus_generations_segments.append(words)
+            else:
+                words = tokenizer.detokenize(sequence_tokens)
+                # else:
+                #     words = []
+                #     for token in sequence_tokens:
+                #         word = tokenizer.tokenizer.decoder[token]
+                #         word = bytearray(
+                #             [tokenizer.tokenizer.byte_decoder[c] for c in word]).decode(
+                #                 'utf-8', errors='replace')
+                #         words.append(word)
+                prompts_plus_generations_segments.append(words)
+
+    if return_segments:
+        return tokens, prompts_plus_generations, prompts_plus_generations_segments
+
+    return tokens, prompts_plus_generations
+
+
+def tokenize_prompts(
+    prompts=None, tokens_to_generate=None, add_BOS=None, rank=0, tokenizer=None, ignore_dist=False, **kwargs
+):
+    """Tokenize prompts and make them avaiable on all ranks."""
+
+    # On all ranks set to None so we can pass them to functions
+    prompts_tokens_cuda_long_tensor = None
+    prompts_length_cuda_long_tensor = None
+
+    # On the specified rank, build the above.
+    attention_mask = None
+    if ignore_dist or torch.distributed.get_rank() == rank:
+        assert prompts is not None
+        assert tokens_to_generate is not None
+        # Tensor of tokens padded and their unpadded length.
+        prompts_tokens_cuda_long_tensor, prompts_length_cuda_long_tensor, attention_mask = _tokenize_prompts_and_batch(
+            prompts, tokens_to_generate, add_BOS, tokenizer, **kwargs
+        )
+
+    return {
+        "input_ids": prompts_tokens_cuda_long_tensor,
+        "attention_mask": attention_mask,
+        # "prompt_length": prompts_length_cuda_long_tensor,
+    }
+
+
+def _tokenize_prompts_and_batch(prompts, tokens_to_generate, add_BOS, tokenizer, **kwargs):
+    """Given a set of prompts and number of tokens to generate:
+    - tokenize prompts
+    - set the sequence length to be the max of length of prompts plus the number of tokens we would like to generate
+    - pad all the sequences to this length so we can convert them into a 2D tensor.
+    """
+
+    # Tokenize all the prompts.
+    # if add_BOS:
+    #     prompts_tokens = [[tokenizer.bos] + tokenizer.tokenize(prompt)
+    #                       for prompt in prompts]
+    # else:
+    #     prompts_tokens = [tokenizer.tokenize(prompt) for prompt in prompts]
+
+    prompts_tokens = [_tokenize_prompt(prompt, tokenizer, add_BOS, **kwargs) for prompt in prompts]
+
+    # Now we have a list of list of tokens which each list has a different
+    # size. We want to extend this list to:
+    #   - incorporate the tokens that need to be generated
+    #   - make all the sequences equal length.
+    # Get the prompts length.
+    prompts_length = [len(prompt_tokens) for prompt_tokens in prompts_tokens]
+    # Get the max prompts length.
+    max_prompt_len = max(prompts_length)
+    # Number of tokens in the each sample of the batch.
+    samples_length = max_prompt_len + tokens_to_generate
+    # Now update the list of list to be of the same size: samples_length.
+    for prompt_tokens, prompt_length in zip(prompts_tokens, prompts_length):
+        padding_size = samples_length - prompt_length
+        prompt_tokens.extend([tokenizer.eos_token_id] * padding_size)
+
+    # Now we are in a structured format, we can convert to tensors.
+    prompts_tokens_tensor = torch.LongTensor(prompts_tokens)
+    prompts_length_tensor = torch.LongTensor(prompts_length)
+    attention_mask = torch.zeros(prompts_tokens_tensor.shape[:2])
+    for i, l in enumerate(prompts_length_tensor):
+        attention_mask[i, :l] = 1
+    return prompts_tokens_tensor, prompts_length_tensor, attention_mask
+
+
+def _tokenize_prompt(prompt, tokenizer, add_BOS=False, media_info={"<image>": 65}, **kwargs):
+    media_tokens = {k: -int(i + 1) for i, k in enumerate(media_info.keys())}
+    media_lengths = media_info.copy()
+
+    if add_BOS:
+        prompt_chunk = [tokenizer.bos_token_id]
+    else:
+        prompt_chunk = []
+
+    # Pure Text
+    if all([media_token not in prompt for media_token in media_tokens.keys()]):
+        enc_chunk = prompt_chunk + tokenizer(prompt, add_special_tokens=False, **kwargs)["input_ids"]
+
+    # Multi-Modal Text
+    else:
+        enc_chunk = prompt_chunk
+        pattern = "|".join(map(re.escape, list(media_tokens.keys())))
+        chunk_strs = re.split(f"({pattern})", prompt)
+        chunk_strs = [x for x in chunk_strs if len(x) > 0]
+        for idx, chunk_str in enumerate(chunk_strs):
+            if chunk_str in media_tokens:
+                enc_chunk += [media_tokens[chunk_str]] * media_lengths[chunk_str]
+            else:
+                tmp_chunk = tokenizer(chunk_str, add_special_tokens=False)["input_ids"]
+                # if idx < len(chunk_strs) - 1: # Last chunk should not have eos
+                #     tmp_chunk += [tokenizer.eod_id]
+                enc_chunk += tmp_chunk
+    return enc_chunk

models/mPLUG_Owl/mplug_owl/tokenization_mplug_owl.py

@@ -0,0 +1,62 @@
+# coding=utf-8
+# Copyright 2022 x-plug and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tokenization classes for MplugOwl."""
+
+from transformers.utils import logging
+from transformers.models.llama.tokenization_llama import LlamaTokenizer
+
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+    "vocab_file": {
+        "MAGAer13/mplug-owl-llama-7b": "https://huggingface.co/MAGAer13/mplug-owl-llama-7b/resolve/main/vocab.txt",
+    },
+}
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+    "MAGAer13/mplug-owl-llama-7b": 1024,
+}
+
+
+class MplugOwlTokenizer(LlamaTokenizer):
+    def __init__(
+        self,
+        vocab_file,
+        unk_token="<unk>",
+        bos_token="<s>",
+        eos_token="</s>",
+        pad_token="<unk>",
+        sp_model_kwargs=None,
+        add_bos_token=False,
+        add_eos_token=False,
+        clean_up_tokenization_spaces=False,
+        **kwargs,
+    ):
+        super().__init__(
+            vocab_file,
+            unk_token,
+            bos_token,
+            eos_token,
+            pad_token,
+            sp_model_kwargs,
+            add_bos_token,
+            add_eos_token,
+            clean_up_tokenization_spaces,
+            **kwargs,
+        )
+        self.eod_id = self.eos_token_id