quoccuongle/MultimodalOCR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add readme (#10)

Browse Source 
* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* Update Readme.md

* remove submodule

* add mPLUG MiniGPT4

* Update Readme.md

* Update Readme.md

* Update Readme.md

---------

Co-authored-by: Yuliang Liu <34134635+Yuliang-Liu@users.noreply.github.com>
This commit is contained in:
lz
2023-06-01 09:57:03 +08:00
committed by GitHub
parent 64f7eb334d
commit 3213a65d96
275 changed files with 16059 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
models/mPLUG_Owl/pipeline/data_utils/processors/__init__.py Normal file
View File

@@ -0,0 +1,9 @@
# Copyright (c) Alibaba. All rights reserved.
from .builder import PROCESSORS, build_processors
from .default_processor import DefaultProcessor
from .caption_processor import CaptionProcessor
__all__ = [
'PROCESSORS', 'build_processors',
'DefaultProcessor', 'CaptionProcessor'
]

12
models/mPLUG_Owl/pipeline/data_utils/processors/builder.py Normal file
View File

@@ -0,0 +1,12 @@
import os
import numpy as np
from data_utils.registry import Registry, build_from_cfg
PROCESSORS = Registry('processors')
def build_processors(processors_cfg):
processors = dict()
for task, processor in processors_cfg.items():
processors[task] = build_from_cfg(processor, PROCESSORS)
return processors

53
models/mPLUG_Owl/pipeline/data_utils/processors/caption_processor.py Normal file
View File

@@ -0,0 +1,53 @@
import torch
from torchvision import transforms
from PIL import Image
import random
from data_utils.randaugment import RandomAugment
from .builder import PROCESSORS
@PROCESSORS.register_module()
class CaptionProcessor:
def __init__(self, image_size=224, min_scale = 0.5, randaug=False):
self.image_size = image_size
self.min_scale = min_scale
if randaug:
self.image_transform = transforms.Compose([
transforms.RandomResizedCrop(image_size,scale=(min_scale, 1.0), interpolation=Image.BICUBIC),
transforms.RandomHorizontalFlip(),
RandomAugment(2,7,isPIL=True,augs=['Identity','AutoContrast','Equalize','Brightness','Sharpness',
'ShearX', 'ShearY', 'TranslateX', 'TranslateY', 'Rotate']),
transforms.ToTensor(),
transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)),
])
else:
self.image_transform = transforms.Compose([
transforms.RandomResizedCrop(image_size,scale=(min_scale, 1.0), interpolation=Image.BICUBIC),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)),
])
self.text_transform = None
def __call__(self, image, text):
assert image or text
if image:
image_input = self.image_transform(image)
else:
image_input = None
if text:
if isinstance(text["prompt"], list):
prompt = random.choice(text["prompt"])
else:
prompt = text["prompt"]
text_input = dict(
prompt=prompt,
completion=text["text"],
)
else:
text_input = None
return image_input, text_input

42
models/mPLUG_Owl/pipeline/data_utils/processors/default_processor.py Normal file
View File

@@ -0,0 +1,42 @@
import torch
from torchvision import transforms
from PIL import Image
import random
from data_utils.randaugment import RandomAugment
from .builder import PROCESSORS
@PROCESSORS.register_module()
class DefaultProcessor:
def __init__(self, image_size=224):
self.image_size = image_size
self.image_transform = transforms.Compose([
transforms.Resize((image_size, image_size),interpolation=Image.BICUBIC),
transforms.ToTensor(),
transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)),
])
self.text_transform = None
def __call__(self, image, text):
assert image or text
if image:
image_input = self.image_transform(image)
else:
image_input = None
if text:
if isinstance(text["prompt"], list):
prompt = random.choice(text["prompt"])
else:
prompt = text["prompt"]
text_input = dict(
prompt=prompt,
completion=text["text"],
)
else:
text_input = None
return image_input, text_input