processing_florence2.py
47.5 KB · 1149 lines · python Raw
1 # coding=utf-8
2 # Copyright 2024 Microsoft and The HuggingFace Inc. team.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 """
16 Processor class for Florence-2.
17 """
18
19 import re
20 import logging
21 from typing import List, Optional, Union
22 import numpy as np
23 import math
24
25 import torch
26
27 from transformers.feature_extraction_utils import BatchFeature
28 from transformers.image_utils import ImageInput, is_valid_image
29 from transformers.processing_utils import ProcessorMixin
30 from transformers.tokenization_utils_base import (
31 PaddingStrategy,
32 PreTokenizedInput,
33 TextInput,
34 TruncationStrategy,
35 )
36 from transformers import BartTokenizer, BartTokenizerFast
37 from transformers.utils import TensorType
38
39
40 logger = logging.getLogger(__name__)
41
42 # Copied from transformers.models.idefics2.processing_idefics2.is_url
43 def is_url(val) -> bool:
44 return isinstance(val, str) and val.startswith("http")
45
46 # Copied from transformers.models.idefics2.processing_idefics2.is_image_or_image_url
47 def is_image_or_image_url(elem):
48 return is_url(elem) or is_valid_image(elem)
49
50
51 def _is_str_or_image(elem):
52 return isinstance(elem, (str)) or is_image_or_image_url(elem)
53
54
55 class Florence2Processor(ProcessorMixin):
56 r"""
57 Constructs a Florence2 processor which wraps a Florence2 image processor and a Florence2 tokenizer into a single processor.
58
59 [`Florence2Processor`] offers all the functionalities of [`CLIPImageProcessor`] and [`BartTokenizerFast`]. See the
60 [`~Florence2Processor.__call__`] and [`~Florence2Processor.decode`] for more information.
61
62 Args:
63 image_processor ([`CLIPImageProcessor`], *optional*):
64 The image processor is a required input.
65 tokenizer ([`BartTokenizerFast`], *optional*):
66 The tokenizer is a required input.
67 """
68
69 attributes = ["image_processor", "tokenizer"]
70 image_processor_class = "CLIPImageProcessor"
71 tokenizer_class = ("BartTokenizer", "BartTokenizerFast")
72
73 def __init__(
74 self,
75 image_processor=None,
76 tokenizer=None,
77 ):
78 if image_processor is None:
79 raise ValueError("You need to specify an `image_processor`.")
80 if tokenizer is None:
81 raise ValueError("You need to specify a `tokenizer`.")
82 if not hasattr(image_processor, "image_seq_length"):
83 raise ValueError("Image processor is missing an `image_seq_length` attribute.")
84
85 self.image_seq_length = image_processor.image_seq_length
86
87 tokens_to_add = {
88 'additional_special_tokens': \
89 tokenizer.additional_special_tokens + \
90 ['<od>', '</od>', '<ocr>', '</ocr>'] + \
91 [f'<loc_{x}>' for x in range(1000)] + \
92 ['<cap>', '</cap>', '<ncap>', '</ncap>','<dcap>', '</dcap>', '<grounding>', '</grounding>', '<seg>', '</seg>', '<sep>', '<region_cap>', '</region_cap>', '<region_to_desciption>', '</region_to_desciption>', '<proposal>', '</proposal>', '<poly>', '</poly>', '<and>']
93 }
94 tokenizer.add_special_tokens(tokens_to_add)
95
96 self.tasks_answer_post_processing_type = {
97 '<OCR>': 'pure_text',
98 '<OCR_WITH_REGION>': 'ocr',
99 '<CAPTION>': 'pure_text',
100 '<DETAILED_CAPTION>': 'pure_text',
101 '<MORE_DETAILED_CAPTION>': 'pure_text',
102 '<OD>': 'description_with_bboxes',
103 '<DENSE_REGION_CAPTION>': 'description_with_bboxes',
104 '<CAPTION_TO_PHRASE_GROUNDING>': "phrase_grounding",
105 '<REFERRING_EXPRESSION_SEGMENTATION>': 'polygons',
106 '<REGION_TO_SEGMENTATION>': 'polygons',
107 '<OPEN_VOCABULARY_DETECTION>': 'description_with_bboxes_or_polygons',
108 '<REGION_TO_CATEGORY>': 'pure_text',
109 '<REGION_TO_DESCRIPTION>': 'pure_text',
110 '<REGION_TO_OCR>': 'pure_text',
111 '<REGION_PROPOSAL>': 'bboxes'
112 }
113
114 self.task_prompts_without_inputs = {
115 '<OCR>': 'What is the text in the image?',
116 '<OCR_WITH_REGION>': 'What is the text in the image, with regions?',
117 '<CAPTION>': 'What does the image describe?',
118 '<DETAILED_CAPTION>': 'Describe in detail what is shown in the image.',
119 '<MORE_DETAILED_CAPTION>': 'Describe with a paragraph what is shown in the image.',
120 '<OD>': 'Locate the objects with category name in the image.',
121 '<DENSE_REGION_CAPTION>': 'Locate the objects in the image, with their descriptions.',
122 '<REGION_PROPOSAL>': 'Locate the region proposals in the image.'
123 }
124
125 self.task_prompts_with_input = {
126 '<CAPTION_TO_PHRASE_GROUNDING>': "Locate the phrases in the caption: {input}",
127 '<REFERRING_EXPRESSION_SEGMENTATION>': 'Locate {input} in the image with mask',
128 '<REGION_TO_SEGMENTATION>': 'What is the polygon mask of region {input}',
129 '<OPEN_VOCABULARY_DETECTION>': 'Locate {input} in the image.',
130 '<REGION_TO_CATEGORY>': 'What is the region {input}?',
131 '<REGION_TO_DESCRIPTION>': 'What does the region {input} describe?',
132 '<REGION_TO_OCR>': 'What text is in the region {input}?',
133 }
134
135 self.post_processor = Florence2PostProcesser(tokenizer=tokenizer)
136
137
138 super().__init__(image_processor, tokenizer)
139
140 def _construct_prompts(self, text):
141 # replace the task tokens with the task prompts if task token is in the text
142 prompts = []
143 for _text in text:
144 # 1. fixed task prompts without additional inputs
145 for task_token, task_prompt in self.task_prompts_without_inputs.items():
146 if task_token in _text:
147 assert _text == task_token, f"Task token {task_token} should be the only token in the text."
148 _text = task_prompt
149 break
150 # 2. task prompts with additional inputs
151 for task_token, task_prompt in self.task_prompts_with_input.items():
152 if task_token in _text:
153 _text = task_prompt.format(input=_text.replace(task_token, ''))
154 break
155 prompts.append(_text)
156 return prompts
157
158 def __call__(
159 self,
160 text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
161 images: ImageInput = None,
162 tokenize_newline_separately: bool = True,
163 padding: Union[bool, str, PaddingStrategy] = False,
164 truncation: Union[bool, str, TruncationStrategy] = None,
165 max_length=None,
166 return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,
167 do_resize: bool = None,
168 do_normalize: bool = None,
169 image_mean: Optional[Union[float, List[float]]] = None,
170 image_std: Optional[Union[float, List[float]]] = None,
171 data_format: Optional["ChannelDimension"] = "channels_first", # noqa: F821
172 input_data_format: Optional[
173 Union[str, "ChannelDimension"] # noqa: F821
174 ] = None,
175 resample: "PILImageResampling" = None, # noqa: F821
176 do_convert_rgb: bool = None,
177 do_thumbnail: bool = None,
178 do_align_long_axis: bool = None,
179 do_rescale: bool = None,
180 ) -> BatchFeature:
181 """
182 Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
183 and `kwargs` arguments to BartTokenizerFast's [`~BartTokenizerFast.__call__`] if `text` is not `None` to encode
184 the text. To prepare the image(s), this method forwards the `images` and `kwrags` arguments to
185 CLIPImageProcessor's [`~CLIPImageProcessor.__call__`] if `images` is not `None`. Please refer to the doctsring
186 of the above two methods for more information.
187
188 Args:
189 text (`str`, `List[str]`, `List[List[str]]`):
190 The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
191 (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
192 `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
193 images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`):
194 The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
195 tensor. In case of a NumPy array/PyTorch tensor, each image should be of shape (C, H, W), where C is a
196 number of channels, H and W are image height and width.
197 tokenize_newline_separately (`bool`, defaults to `True`):
198 Adds a separately tokenized '\n' at the end of the prompt.
199 padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
200 Select a strategy to pad the returned sequences (according to the model's padding side and padding
201 index) among:
202 - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
203 sequence if provided).
204 - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
205 acceptable input length for the model if that argument is not provided.
206 - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
207 lengths).
208 max_length (`int`, *optional*):
209 Maximum length of the returned list and optionally padding length (see above).
210 truncation (`bool`, *optional*):
211 Activates truncation to cut input sequences longer than `max_length` to `max_length`.
212 return_tensors (`str` or [`~utils.TensorType`], *optional*):
213 If set, will return tensors of a particular framework. Acceptable values are:
214
215 - `'tf'`: Return TensorFlow `tf.constant` objects.
216 - `'pt'`: Return PyTorch `torch.Tensor` objects.
217 - `'np'`: Return NumPy `np.ndarray` objects.
218 - `'jax'`: Return JAX `jnp.ndarray` objects.
219
220 Returns:
221 [`BatchFeature`]: A [`BatchFeature`] with the following fields:
222
223 - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`. If `suffix`
224 is provided, the `input_ids` will also contain the suffix input ids.
225 - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
226 `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
227 `None`).
228 - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
229 - **labels** -- Labels compatible with training if `suffix` is not None
230 """
231
232 return_token_type_ids = False
233
234 if images is None:
235 raise ValueError("`images` are expected as arguments to a `Florence2Processor` instance.")
236 if text is None:
237 logger.warning_once(
238 "You are using Florence-2 without a text prompt."
239 )
240 text = ""
241
242 if isinstance(text, List) and isinstance(images, List):
243 if len(images) < len(text):
244 raise ValueError(
245 f"Received {len(images)} images for {len(text)} prompts. Each prompt should be associated with an image."
246 )
247 if _is_str_or_image(text):
248 text = [text]
249 elif isinstance(text, list) and _is_str_or_image(text[0]):
250 pass
251
252 pixel_values = self.image_processor(
253 images,
254 do_resize=do_resize,
255 do_normalize=do_normalize,
256 return_tensors=return_tensors,
257 image_mean=image_mean,
258 image_std=image_std,
259 input_data_format=input_data_format,
260 data_format=data_format,
261 resample=resample,
262 do_convert_rgb=do_convert_rgb,
263 )["pixel_values"]
264
265 if max_length is not None:
266 max_length -= self.image_seq_length # max_length has to account for the image tokens
267
268 text = self._construct_prompts(text)
269
270 inputs = self.tokenizer(
271 text,
272 return_tensors=return_tensors,
273 padding=padding,
274 max_length=max_length,
275 truncation=truncation,
276 return_token_type_ids=return_token_type_ids,
277 )
278
279 return_data = {**inputs, "pixel_values": pixel_values}
280
281 if return_token_type_ids:
282 labels = inputs["input_ids"].masked_fill(inputs["token_type_ids"] == 0, -100)
283 return_data.update({"labels": labels})
284 return BatchFeature(data=return_data)
285
286 # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode with CLIP->Florence2
287 def batch_decode(self, *args, **kwargs):
288 """
289 This method forwards all its arguments to BartTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
290 refer to the docstring of this method for more information.
291 """
292 return self.tokenizer.batch_decode(*args, **kwargs)
293
294 # Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode with CLIP->Florence2
295 def decode(self, *args, **kwargs):
296 """
297 This method forwards all its arguments to BartTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
298 the docstring of this method for more information.
299 """
300 return self.tokenizer.decode(*args, **kwargs)
301
302 @property
303 # Copied from transformers.models.clip.processing_clip.CLIPProcessor.model_input_names with CLIP->Florence2
304 def model_input_names(self):
305 tokenizer_input_names = self.tokenizer.model_input_names
306 image_processor_input_names = self.image_processor.model_input_names
307 return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
308
309 def post_process_generation(self, text=None, sequence=None, transition_beam_score=None, task=None, image_size=None):
310 """
311 Post-process the output of the model to each of the task outputs.
312
313 Args:
314 text (`str`): The text to post-process.
315 task (`str`): The task to post-process the text for.
316 image_size (`Tuple[int, int]`): The size of the image. height x width.
317 """
318
319 task_answer_post_processing_type = self.tasks_answer_post_processing_type.get(task, 'pure_text')
320 task_answer = self.post_processor(
321 text=text,
322 sequence=sequence,
323 transition_beam_score=transition_beam_score,
324 image_size=image_size,
325 parse_tasks=task_answer_post_processing_type,
326 )[task_answer_post_processing_type]
327
328 if task_answer_post_processing_type == 'pure_text':
329 final_answer = task_answer
330 # remove the special tokens
331 final_answer = final_answer.replace('<s>', '').replace('</s>', '')
332 elif task_answer_post_processing_type in ['od', 'description_with_bboxes', 'bboxes']:
333 od_instances = task_answer
334 bboxes_od = [_od_instance['bbox'] for _od_instance in od_instances]
335 labels_od = [str(_od_instance['cat_name']) for _od_instance in od_instances]
336 final_answer = {'bboxes': bboxes_od, 'labels': labels_od}
337 if len(od_instances) and 'score' in od_instances[0]:
338 scores_od = [_od_instance['score'] for _od_instance in od_instances]
339 final_answer['scores'] = scores_od
340 elif task_answer_post_processing_type in ['ocr']:
341 bboxes = [_od_instance['quad_box'] for _od_instance in task_answer]
342 labels = [str(_od_instance['text']) for _od_instance in task_answer]
343 final_answer = {'quad_boxes': bboxes, 'labels': labels}
344 elif task_answer_post_processing_type in ['phrase_grounding']:
345 bboxes = []
346 labels = []
347 for _grounded_phrase in task_answer:
348 for _bbox in _grounded_phrase['bbox']:
349 bboxes.append(_bbox)
350 labels.append(_grounded_phrase['cat_name'])
351 final_answer = {'bboxes': bboxes, 'labels': labels}
352 elif task_answer_post_processing_type in ['description_with_polygons', 'polygons']:
353 labels = []
354 polygons = []
355 for result in task_answer:
356 label = result['cat_name']
357 _polygons = result['polygons']
358 labels.append(label)
359 polygons.append(_polygons)
360 final_answer = {'polygons': polygons, 'labels': labels}
361 elif task_answer_post_processing_type in ['description_with_bboxes_or_polygons']:
362 bboxes = []
363 bboxes_labels = []
364 polygons = []
365 polygons_labels = []
366 for result in task_answer:
367 label = result['cat_name']
368 if 'polygons' in result:
369 _polygons = result['polygons']
370 polygons.append(_polygons)
371 polygons_labels.append(label)
372 else:
373 _bbox = result['bbox']
374 bboxes.append(_bbox)
375 bboxes_labels.append(label)
376 final_answer = {'bboxes': bboxes, 'bboxes_labels': bboxes_labels, 'polygons': polygons, 'polygons_labels': polygons_labels}
377 else:
378 raise ValueError('Unknown task answer post processing type: {}'.format(task_answer_post_processing_type))
379
380 final_answer = {
381 task: final_answer}
382 return final_answer
383
384 class BoxQuantizer(object):
385 def __init__(self, mode, bins):
386 self.mode = mode
387 self.bins = bins
388
389 def quantize(self, boxes: torch.Tensor, size):
390 bins_w, bins_h = self.bins # Quantization bins.
391 size_w, size_h = size # Original image size.
392 size_per_bin_w = size_w / bins_w
393 size_per_bin_h = size_h / bins_h
394 xmin, ymin, xmax, ymax = boxes.split(1, dim=-1) # Shape: 4 * [N, 1].
395
396 if self.mode == 'floor':
397 quantized_xmin = (
398 xmin / size_per_bin_w).floor().clamp(0, bins_w - 1)
399 quantized_ymin = (
400 ymin / size_per_bin_h).floor().clamp(0, bins_h - 1)
401 quantized_xmax = (
402 xmax / size_per_bin_w).floor().clamp(0, bins_w - 1)
403 quantized_ymax = (
404 ymax / size_per_bin_h).floor().clamp(0, bins_h - 1)
405
406 elif self.mode == 'round':
407 raise NotImplementedError()
408
409 else:
410 raise ValueError('Incorrect quantization type.')
411
412 quantized_boxes = torch.cat(
413 (quantized_xmin, quantized_ymin, quantized_xmax, quantized_ymax), dim=-1
414 ).int()
415
416 return quantized_boxes
417
418 def dequantize(self, boxes: torch.Tensor, size):
419 bins_w, bins_h = self.bins # Quantization bins.
420 size_w, size_h = size # Original image size.
421 size_per_bin_w = size_w / bins_w
422 size_per_bin_h = size_h / bins_h
423 xmin, ymin, xmax, ymax = boxes.split(1, dim=-1) # Shape: 4 * [N, 1].
424
425 if self.mode == 'floor':
426 # Add 0.5 to use the center position of the bin as the coordinate.
427 dequantized_xmin = (xmin + 0.5) * size_per_bin_w
428 dequantized_ymin = (ymin + 0.5) * size_per_bin_h
429 dequantized_xmax = (xmax + 0.5) * size_per_bin_w
430 dequantized_ymax = (ymax + 0.5) * size_per_bin_h
431
432 elif self.mode == 'round':
433 raise NotImplementedError()
434
435 else:
436 raise ValueError('Incorrect quantization type.')
437
438 dequantized_boxes = torch.cat(
439 (dequantized_xmin, dequantized_ymin,
440 dequantized_xmax, dequantized_ymax), dim=-1
441 )
442
443 return dequantized_boxes
444
445
446 class CoordinatesQuantizer(object):
447 """
448 Quantize coornidates (Nx2)
449 """
450
451 def __init__(self, mode, bins):
452 self.mode = mode
453 self.bins = bins
454
455 def quantize(self, coordinates: torch.Tensor, size):
456 bins_w, bins_h = self.bins # Quantization bins.
457 size_w, size_h = size # Original image size.
458 size_per_bin_w = size_w / bins_w
459 size_per_bin_h = size_h / bins_h
460 assert coordinates.shape[-1] == 2, 'coordinates should be shape (N, 2)'
461 x, y = coordinates.split(1, dim=-1) # Shape: 4 * [N, 1].
462
463 if self.mode == 'floor':
464 quantized_x = (x / size_per_bin_w).floor().clamp(0, bins_w - 1)
465 quantized_y = (y / size_per_bin_h).floor().clamp(0, bins_h - 1)
466
467 elif self.mode == 'round':
468 raise NotImplementedError()
469
470 else:
471 raise ValueError('Incorrect quantization type.')
472
473 quantized_coordinates = torch.cat(
474 (quantized_x, quantized_y), dim=-1
475 ).int()
476
477 return quantized_coordinates
478
479 def dequantize(self, coordinates: torch.Tensor, size):
480 bins_w, bins_h = self.bins # Quantization bins.
481 size_w, size_h = size # Original image size.
482 size_per_bin_w = size_w / bins_w
483 size_per_bin_h = size_h / bins_h
484 assert coordinates.shape[-1] == 2, 'coordinates should be shape (N, 2)'
485 x, y = coordinates.split(1, dim=-1) # Shape: 4 * [N, 1].
486
487 if self.mode == 'floor':
488 # Add 0.5 to use the center position of the bin as the coordinate.
489 dequantized_x = (x + 0.5) * size_per_bin_w
490 dequantized_y = (y + 0.5) * size_per_bin_h
491
492 elif self.mode == 'round':
493 raise NotImplementedError()
494
495 else:
496 raise ValueError('Incorrect quantization type.')
497
498 dequantized_coordinates = torch.cat(
499 (dequantized_x, dequantized_y), dim=-1
500 )
501
502 return dequantized_coordinates
503
504
505 class Florence2PostProcesser(object):
506 r"""
507 Florence-2 post process for converting text prediction to various tasks results.
508
509 Args:
510 config: A dict of configs.
511 tokenizer: A tokenizer for decoding text to spans.
512 sample config:
513 UNIFIED_POST_PROCESS:
514 # commom configs
515 NUM_BBOX_HEIGHT_BINS: 1000
516 NUM_BBOX_WIDTH_BINS: 1000
517 COORDINATES_HEIGHT_BINS: 1000
518 COORDINATES_WIDTH_BINS: 1000
519 # task specific configs, override the common configs
520 PRASE_TASKS:
521 - TASK_NAME: 'video_dense_caption'
522 PATTERN: 'r<time_(\d+)><time_(\d+)>([a-zA-Z0-9 ]+)'
523 SCORE_MODE: 'avg_cat_name_scores'
524 NUM_BINS: 100
525 - TASK_NAME: 'od'
526 PATTERN: 'r<loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)>([a-zA-Z0-9 ]+)'
527 SCORE_MODE: 'avg_cat_name_scores'
528
529 Returns:
530 parsed_dict (dict): A dict of parsed results.
531 """
532 def __init__(
533 self,
534 tokenizer=None
535 ):
536 parse_tasks = []
537 parse_task_configs = {}
538 config = self._create_default_config()
539 for task in config['PARSE_TASKS']:
540 parse_tasks.append(task['TASK_NAME'])
541 parse_task_configs[task['TASK_NAME']] = task
542
543 self.config = config
544 self.parse_tasks = parse_tasks
545 self.parse_tasks_configs = parse_task_configs
546
547 self.tokenizer = tokenizer
548 if self.tokenizer is not None:
549 self.all_special_tokens = set(self.tokenizer.all_special_tokens)
550
551 self.init_quantizers()
552 self.black_list_of_phrase_grounding = self._create_black_list_of_phrase_grounding()
553
554 def _create_black_list_of_phrase_grounding(self):
555 black_list = {}
556
557 if 'phrase_grounding' in self.parse_tasks and self.parse_tasks_configs['phrase_grounding']['FILTER_BY_BLACK_LIST']:
558 black_list = set(
559 ['it', 'I', 'me', 'mine',
560 'you', 'your', 'yours',
561 'he', 'him', 'his',
562 'she', 'her', 'hers',
563 'they', 'them', 'their', 'theirs',
564 'one', 'oneself',
565 'we', 'us', 'our', 'ours',
566 'you', 'your', 'yours',
567 'they', 'them', 'their', 'theirs',
568 'mine', 'yours', 'his', 'hers', 'its',
569 'ours', 'yours', 'theirs',
570 'myself', 'yourself', 'himself', 'herself', 'itself',
571 'ourselves', 'yourselves', 'themselves',
572 'this', 'that',
573 'these', 'those',
574 'who', 'whom', 'whose', 'which', 'what',
575 'who', 'whom', 'whose', 'which', 'that',
576 'all', 'another', 'any', 'anybody', 'anyone', 'anything',
577 'each', 'everybody', 'everyone', 'everything',
578 'few', 'many', 'nobody', 'none', 'one', 'several',
579 'some', 'somebody', 'someone', 'something',
580 'each other', 'one another',
581 'myself', 'yourself', 'himself', 'herself', 'itself',
582 'ourselves', 'yourselves', 'themselves',
583 'the image', 'image', 'images', 'the', 'a', 'an', 'a group',
584 'other objects', 'lots', 'a set',
585 ]
586 )
587
588 return black_list
589
590 def _create_default_config(self):
591 config = {
592 'NUM_BBOX_HEIGHT_BINS': 1000,
593 'NUM_BBOX_WIDTH_BINS': 1000,
594 'BOX_QUANTIZATION_MODE': 'floor',
595 'COORDINATES_HEIGHT_BINS': 1000,
596 'COORDINATES_WIDTH_BINS': 1000,
597 'COORDINATES_QUANTIZATION_MODE': 'floor',
598 'PARSE_TASKS': [
599 {
600 'TASK_NAME': 'od',
601 'PATTERN': r'([a-zA-Z0-9 ]+)<loc_(\\d+)><loc_(\\d+)><loc_(\\d+)><loc_(\\d+)>',
602 'SCORE_MODE': 'avg_loc_scores'
603 },
604 {
605 'TASK_NAME': 'ocr',
606 'PATTERN': r'(.+?)<loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)>',
607 'AREA_THRESHOLD': 0.00
608 },
609 {
610 'TASK_NAME': 'phrase_grounding',
611 'FILTER_BY_BLACK_LIST': True
612 },
613 {
614 'TASK_NAME': 'pure_text',
615 },
616 {
617 'TASK_NAME': 'description_with_bboxes',
618 'SCORE_MODE': 'avg_loc_scores'
619 },
620 {
621 'TASK_NAME': 'description_with_polygons',
622 },
623 {
624 'TASK_NAME': 'polygons',
625 },
626 {
627 'TASK_NAME': 'bboxes',
628 },
629 {
630 'TASK_NAME': 'description_with_bboxes_or_polygons',
631 }
632 ]
633 }
634
635 return config
636
637 def init_quantizers(self):
638 # we have box_quantizer (od, grounding) and coordinates_quantizer (ocr, referring_segmentation)
639 num_bbox_height_bins = self.config.get('NUM_BBOX_HEIGHT_BINS', 1000)
640 num_bbox_width_bins = self.config.get('NUM_BBOX_WIDTH_BINS', 1000)
641 box_quantization_mode = self.config.get('BOX_QUANTIZATION_MODE', 'floor')
642 self.box_quantizer = BoxQuantizer(
643 box_quantization_mode,
644 (num_bbox_width_bins, num_bbox_height_bins),
645 )
646
647 num_bbox_height_bins = self.config['COORDINATES_HEIGHT_BINS'] if 'COORDINATES_HEIGHT_BINS' in self.config else self.config.get('NUM_BBOX_HEIGHT_BINS', 1000)
648 num_bbox_width_bins = self.config['COORDINATES_WIDTH_BINS'] if 'COORDINATES_WIDTH_BINS' in self.config else self.config.get('NUM_BBOX_WIDTH_BINS', 1000)
649 box_quantization_mode = self.config.get('COORDINATES_QUANTIZATION_MODE') if 'COORDINATES_QUANTIZATION_MODE' in self.config else self.config.get('BOX_QUANTIZATION_MODE', 'floor')
650 self.coordinates_quantizer = CoordinatesQuantizer(
651 box_quantization_mode,
652 (num_bbox_width_bins, num_bbox_height_bins),
653 )
654
655 def decode_with_spans(self, tokenizer, token_ids):
656 filtered_tokens = tokenizer.convert_ids_to_tokens(
657 token_ids, skip_special_tokens=False)
658 assert len(filtered_tokens) == len(token_ids)
659
660 sub_texts = []
661 for token in filtered_tokens:
662 if token in self.all_special_tokens:
663 sub_texts.append(token)
664 else:
665 if isinstance(tokenizer, (BartTokenizer, BartTokenizerFast)):
666 sub_text = tokenizer.convert_tokens_to_string([token])
667 else:
668 raise ValueError(f'type {type(tokenizer)} not supported')
669 sub_texts.append(sub_text)
670
671 text = ''
672 spans = []
673 for sub_text in sub_texts:
674 span = (len(text), len(text) + len(sub_text)) # [start index, end index).
675 text += sub_text
676 spans.append(span)
677
678 return text, spans
679
680 def parse_od_from_text_and_spans(
681 self,
682 text,
683 pattern,
684 image_size,
685 phrase_centric=False
686 ):
687 parsed = list(re.finditer(pattern, text))
688
689 instances = []
690 for i in range(len(parsed)):
691 # Prepare instance.
692 instance = {}
693
694 if phrase_centric:
695 bbox_bins = [int(parsed[i].group(j)) for j in range(2, 6)]
696 else:
697 bbox_bins = [int(parsed[i].group(j)) for j in range(1, 5)]
698 instance['bbox'] = self.box_quantizer.dequantize(
699 boxes=torch.tensor(bbox_bins),
700 size=image_size
701 ).tolist()
702
703 if phrase_centric:
704 instance['cat_name'] = parsed[i].group(1).lower().strip()
705 else:
706 instance['cat_name'] = parsed[i].group(5).lower().strip()
707 instances.append(instance)
708
709 return instances
710
711 def parse_ocr_from_text_and_spans(self,
712 text,
713 pattern,
714 image_size,
715 area_threshold=-1.0,
716 ):
717 bboxes = []
718 labels = []
719 text = text.replace('<s>', '')
720 # ocr with regions
721 parsed = re.findall(pattern, text)
722 instances = []
723 image_width, image_height = image_size
724
725 for ocr_line in parsed:
726 ocr_content = ocr_line[0]
727 quad_box = ocr_line[1:]
728 quad_box = [int(i) for i in quad_box]
729 quad_box = self.coordinates_quantizer.dequantize(
730 torch.tensor(np.array(quad_box).reshape(-1, 2)),
731 size=image_size
732 ).reshape(-1).tolist()
733
734 if area_threshold > 0:
735 x_coords = [i for i in quad_box[0::2]]
736 y_coords = [i for i in quad_box[1::2]]
737
738 # apply the Shoelace formula
739 area = 0.5 * abs(sum(x_coords[i] * y_coords[i + 1] - x_coords[i + 1] * y_coords[i] for i in range(4 - 1)))
740
741 if area < (image_width * image_height) * area_threshold:
742 continue
743
744 bboxes.append(quad_box)
745 labels.append(ocr_content)
746 instances.append({
747 'quad_box': quad_box,
748 'text': ocr_content,
749 })
750 return instances
751
752 def parse_phrase_grounding_from_text_and_spans(self, text, pattern, image_size):
753 # ignore <s> </s> and <pad>
754 cur_span = 0
755 if text.startswith('<s>'):
756 cur_span += 3
757
758 text = text.replace('<s>', '')
759 text = text.replace('</s>', '')
760 text = text.replace('<pad>', '')
761
762 pattern = r"([^<]+(?:<loc_\d+>){4,})"
763 phrases = re.findall(pattern, text)
764
765 # pattern should be text pattern and od pattern
766 pattern = r'^\s*(.*?)(?=<od>|</od>|<box>|</box>|<bbox>|</bbox>|<loc_)'
767 box_pattern = r'<loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)>'
768
769 instances = []
770 for pharse_text in phrases:
771 phrase_text_strip = pharse_text.replace('<ground>', '', 1)
772 phrase_text_strip = pharse_text.replace('<obj>', '', 1)
773
774 if phrase_text_strip == '':
775 cur_span += len(pharse_text)
776 continue
777
778 # Prepare instance.
779 instance = {}
780
781 # parse phrase, get string
782 phrase = re.search(pattern, phrase_text_strip)
783 if phrase is None:
784 cur_span += len(pharse_text)
785 continue
786
787 # parse bboxes by box_pattern
788 bboxes_parsed = list(re.finditer(box_pattern, pharse_text))
789 if len(bboxes_parsed) == 0:
790 cur_span += len(pharse_text)
791 continue
792
793 phrase = phrase.group()
794 # remove leading and trailing spaces
795 phrase = phrase.strip()
796
797 if phrase in self.black_list_of_phrase_grounding:
798 cur_span += len(pharse_text)
799 continue
800
801 # a list of list
802 bbox_bins = [[int(_bboxes_parsed.group(j)) for j in range(1, 5)] for _bboxes_parsed in bboxes_parsed]
803 instance['bbox'] = self.box_quantizer.dequantize(
804 boxes=torch.tensor(bbox_bins),
805 size=image_size
806 ).tolist()
807
808 # exclude non-ascii characters
809 phrase = phrase.encode('ascii',errors='ignore').decode('ascii')
810 instance['cat_name'] = phrase
811
812 instances.append(instance)
813
814 return instances
815
816 def parse_description_with_bboxes_from_text_and_spans(
817 self,
818 text,
819 spans=None,
820 scores=None,
821 score_mode=None,
822 pattern=None,
823 image_size=None,
824 allow_empty_phrase=False
825 ):
826 def find_matched_token_indices(cur_span, token_spans):
827 inds = []
828 for i, token_span in enumerate(token_spans):
829 if not (token_span[1] <= cur_span[0] or token_span[0] >= cur_span[1]):
830 inds.append(i)
831 return inds
832
833 cur_span = 0
834 if text.startswith('<s>'):
835 cur_span += 3
836
837 text = text.replace('<s>', '')
838 text = text.replace('</s>', '')
839 text = text.replace('<pad>', '')
840
841 if allow_empty_phrase:
842 pattern = rf"(?:(?:<loc_\d+>){{4,}})"
843 else:
844 pattern = r"([^<]+(?:<loc_\d+>){4,})"
845 phrases = re.findall(pattern, text)
846
847 # pattern should be text pattern and od pattern
848 pattern = r'^\s*(.*?)(?=<od>|</od>|<box>|</box>|<bbox>|</bbox>|<loc_)'
849 box_pattern = r'<loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)>'
850
851 instances = []
852 for pharse_text in phrases:
853 phrase_text_strip = pharse_text.replace('<ground>', '', 1)
854 phrase_text_strip = pharse_text.replace('<obj>', '', 1)
855
856 if phrase_text_strip == '' and not allow_empty_phrase:
857 cur_span += len(pharse_text)
858 continue
859
860 # parse phrase, get string
861 phrase = re.search(pattern, phrase_text_strip)
862 if phrase is None:
863 cur_span += len(pharse_text)
864 continue
865
866 phrase_span = phrase.span()
867 phrase = phrase.group()
868 # remove leading and trailing spaces
869 phrase = phrase.strip()
870
871 # parse bboxes by box_pattern
872 bboxes_parsed = list(re.finditer(box_pattern, pharse_text))
873 if len(bboxes_parsed) == 0:
874 cur_span += len(pharse_text)
875 continue
876
877 # a list of list
878 bbox_bins = [[int(_bboxes_parsed.group(j)) for j in range(1, 5)] for _bboxes_parsed in bboxes_parsed]
879
880 bboxes = self.box_quantizer.dequantize(
881 boxes=torch.tensor(bbox_bins),
882 size=image_size
883 ).tolist()
884
885 if score_mode == 'avg_loc_scores':
886 if spans is None or scores is None:
887 all_scores = None
888 else:
889 bbox_end_spans = [_bboxes_parsed.span(0) for _bboxes_parsed in bboxes_parsed]
890 all_scores = []
891 for _spans in bbox_end_spans:
892 token_inds = find_matched_token_indices((_spans[0] + cur_span, _spans[1]+ cur_span), spans)
893 loc_scores = [scores[token_i] for token_i in token_inds]
894 score = sum(loc_scores) / len(loc_scores)
895 all_scores.append(score)
896 elif score_mode == 'avg_cat_name_scores':
897 if spans is None or scores is None:
898 all_scores = None
899 else:
900 cat_name_token_inds = find_matched_token_indices((phrase_span[0] + cur_span, phrase_span[1]+cur_span), spans)
901 cat_name_scores = [scores[token_i] for token_i in cat_name_token_inds]
902 score = sum(cat_name_scores) / len(cat_name_scores)
903 all_scores = [score] * len(bboxes)
904 elif score_mode is None:
905 all_scores = None
906 else:
907 raise ValueError('Unknown score mode: {}'.format(score_mode))
908
909 phrase = phrase.encode('ascii',errors='ignore').decode('ascii')
910 for _idx, _bboxes in enumerate(bboxes):
911 # Prepare instance.
912 instance = {}
913 instance['bbox'] = _bboxes
914 # exclude non-ascii characters
915 instance['cat_name'] = phrase
916 if all_scores is not None:
917 instance['score'] = math.exp(all_scores[_idx])
918 instances.append(instance)
919
920 cur_span += len(pharse_text)
921
922 return instances
923
924 def parse_description_with_polygons_from_text_and_spans(self, text, pattern, image_size,
925 allow_empty_phrase=False,
926 polygon_sep_token='<sep>',
927 polygon_start_token='<poly>',
928 polygon_end_token='</poly>',
929 with_box_at_start=False,
930 ):
931
932 # ref_seg format: '<expression><x1><y1><x2><y2><><><sep><><><><>'
933 # ignore <s> </s> and <pad>
934
935 text = text.replace('<s>', '')
936 text = text.replace('</s>', '')
937 text = text.replace('<pad>', '')
938
939 if allow_empty_phrase:
940 pattern = rf"(?:(?:<loc_\d+>|{re.escape(polygon_sep_token)}|{re.escape(polygon_start_token)}|{re.escape(polygon_end_token)}){{4,}})"
941 else:
942 # [^<]+: This part matches one or more characters that are not the < symbol.
943 # The ^ inside the square brackets [] is a negation, meaning it matches anything except <.
944 #
945 pattern = rf"([^<]+(?:<loc_\d+>|{re.escape(polygon_sep_token)}|{re.escape(polygon_start_token)}|{re.escape(polygon_end_token)}){{4,}})"
946 phrases = re.findall(pattern, text)
947
948 phrase_string_pattern = r'^\s*(.*?)(?=<od>|</od>|<box>|</box>|<bbox>|</bbox>|<loc_|<poly>)'
949 box_pattern = rf'((?:<loc_\d+>)+)(?:{re.escape(polygon_sep_token)}|$)'
950
951 # one polygons instance is separated by polygon_start_token and polygon_end_token
952 polygons_instance_pattern = rf'{re.escape(polygon_start_token)}(.*?){re.escape(polygon_end_token)}'
953
954 instances = []
955 for phrase_text in phrases:
956
957 # exclude loc_\d+>
958 # need to get span if want to include category score
959 phrase_text_strip = re.sub(r'^loc_\d+>', '', phrase_text, count=1)
960
961 # phrase = phrase.replace('<poly>', '')
962 # phrase = phrase.replace('poly>', '')
963
964 if phrase_text_strip == '' and not allow_empty_phrase:
965 continue
966
967
968 # parse phrase, get string
969 phrase = re.search(phrase_string_pattern, phrase_text_strip)
970 if phrase is None:
971 continue
972 phrase = phrase.group()
973 # remove leading and trailing spaces
974 phrase = phrase.strip()
975
976 # parse bboxes by box_pattern
977
978 # split by polygon_start_token and polygon_end_token first using polygons_instance_pattern
979 if polygon_start_token in phrase_text and polygon_end_token in phrase_text:
980 polygons_instances_parsed = list(re.finditer(polygons_instance_pattern, phrase_text))
981 else:
982 polygons_instances_parsed = [phrase_text]
983
984 for _polygons_instances_parsed in polygons_instances_parsed:
985 # Prepare instance.
986 instance = {}
987
988 # polygons_parsed= list(re.finditer(box_pattern, phrase_text))
989 if isinstance(_polygons_instances_parsed, str):
990 polygons_parsed= list(re.finditer(box_pattern, _polygons_instances_parsed))
991 else:
992 polygons_parsed= list(re.finditer(box_pattern, _polygons_instances_parsed.group(1)))
993 if len(polygons_parsed) == 0:
994 continue
995
996 # a list of list (polygon)
997 bbox = []
998 polygons = []
999 for _polygon_parsed in polygons_parsed:
1000 # group 1: whole <loc_\d+>...</loc_\d+>
1001 _polygon = _polygon_parsed.group(1)
1002 # parse into list of int
1003 _polygon = [int(_loc_parsed.group(1)) for _loc_parsed in re.finditer(r'<loc_(\d+)>', _polygon)]
1004 if with_box_at_start and len(bbox) == 0:
1005 if len(_polygon) > 4:
1006 # no valid bbox prediction
1007 bbox = _polygon[:4]
1008 _polygon = _polygon[4:]
1009 else:
1010 bbox = [0, 0, 0, 0]
1011 # abandon last element if is not paired
1012 if len(_polygon) % 2 == 1:
1013 _polygon = _polygon[:-1]
1014
1015 # reshape into (n, 2)
1016 _polygon = self.coordinates_quantizer.dequantize(
1017 torch.tensor(np.array(_polygon).reshape(-1, 2)),
1018 size=image_size
1019 ).reshape(-1).tolist()
1020 # reshape back
1021 polygons.append(_polygon)
1022
1023 instance['cat_name'] = phrase
1024 instance['polygons'] = polygons
1025 if len(bbox) != 0:
1026 instance['bbox'] = self.box_quantizer.dequantize(
1027 boxes=torch.tensor([bbox]),
1028 size=image_size
1029 ).tolist()[0]
1030
1031 instances.append(instance)
1032
1033 return instances
1034
1035 def __call__(
1036 self,
1037 text=None,
1038 sequence=None,
1039 transition_beam_score=None,
1040 image_size=None,
1041 parse_tasks=None,
1042 ):
1043 """
1044 Args:
1045 text: model outputs
1046 image_size: (width, height)
1047 parse_tasks: a list of tasks to parse, if None, parse all tasks.
1048 """
1049 if parse_tasks is not None:
1050 if isinstance(parse_tasks, str):
1051 parse_tasks = [parse_tasks]
1052 for _parse_task in parse_tasks:
1053 assert _parse_task in self.parse_tasks, f'parse task {_parse_task} not supported'
1054
1055 # sequence or text should be provided
1056 assert sequence is not None or text is not None, 'sequence or text should be provided'
1057 assert sequence is None or text is None, 'only one of sequence and text should be provided'
1058
1059 if sequence is not None:
1060 sequence = sequence.tolist()[1:]
1061 text, spans = self.decode_with_spans(self.tokenizer, sequence)
1062 if transition_beam_score is not None:
1063 transition_beam_score = transition_beam_score.tolist()
1064 assert len(sequence) == len(transition_beam_score)
1065 else:
1066 spans = None
1067 transition_beam_score = None
1068
1069 parsed_dict = {
1070 'text': text
1071 }
1072
1073 for task in self.parse_tasks:
1074 if parse_tasks is not None and task not in parse_tasks:
1075 continue
1076
1077 pattern = self.parse_tasks_configs[task].get('PATTERN', None)
1078 score_mode = self.parse_tasks_configs[task].get('SCORE_MODE', None)
1079
1080 if task == 'ocr':
1081 instances = self.parse_ocr_from_text_and_spans(
1082 text,
1083 pattern=pattern,
1084 image_size=image_size,
1085 area_threshold=self.parse_tasks_configs[task].get('AREA_THRESHOLD', 0.0),
1086 )
1087 parsed_dict['ocr'] = instances
1088 elif task == 'phrase_grounding':
1089 instances = self.parse_phrase_grounding_from_text_and_spans(
1090 text,
1091 pattern=pattern,
1092 image_size=image_size,
1093 )
1094 parsed_dict['phrase_grounding'] = instances
1095 elif task == 'pure_text':
1096 parsed_dict['pure_text'] = text
1097 elif task == 'description_with_bboxes':
1098 instances = self.parse_description_with_bboxes_from_text_and_spans(
1099 text,
1100 spans=spans,
1101 scores=transition_beam_score,
1102 score_mode=score_mode,
1103 pattern=pattern,
1104 image_size=image_size,
1105 )
1106 parsed_dict['description_with_bboxes'] = instances
1107 elif task == 'description_with_polygons':
1108 instances = self.parse_description_with_polygons_from_text_and_spans(
1109 text,
1110 pattern=pattern,
1111 image_size=image_size,
1112 )
1113 parsed_dict['description_with_polygons'] = instances
1114 elif task == 'polygons':
1115 instances = self.parse_description_with_polygons_from_text_and_spans(
1116 text,
1117 pattern=pattern,
1118 image_size=image_size,
1119 allow_empty_phrase=True,
1120 )
1121 parsed_dict['polygons'] = instances
1122 elif task == 'bboxes':
1123 instances = self.parse_description_with_bboxes_from_text_and_spans(
1124 text,
1125 pattern=pattern,
1126 image_size=image_size,
1127 allow_empty_phrase=True,
1128 )
1129 parsed_dict['bboxes'] = instances
1130 elif task == 'description_with_bboxes_or_polygons':
1131 if '<poly>' in text:
1132 # only support either polygons or bboxes, not both at the same time
1133 instances = self.parse_description_with_polygons_from_text_and_spans(
1134 text,
1135 pattern=pattern,
1136 image_size=image_size,
1137 )
1138 else:
1139 instances = self.parse_description_with_bboxes_from_text_and_spans(
1140 text,
1141 pattern=pattern,
1142 image_size=image_size,
1143 )
1144 parsed_dict['description_with_bboxes_or_polygons'] = instances
1145 else:
1146 raise ValueError("task {} is not supported".format(task))
1147
1148 return parsed_dict
1149