modeling_vit.py
20.8 KB · 616 lines · python Raw
1 # Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
2 #
3 # NVIDIA CORPORATION and its licensors retain all intellectual property
4 # and proprietary rights in and to this software, related documentation
5 # and any modifications thereto. Any use, reproduction, disclosure or
6 # distribution of this software and related documentation without an express
7 # license agreement from NVIDIA CORPORATION is strictly prohibited.
8
9 import math
10 from copy import deepcopy
11 from typing import Union, Tuple, Sequence, Optional, List
12
13 import torch
14 import torch.nn as nn
15 import torch.nn.functional as F
16 try:
17 from transformers.activations import PytorchGELUTanh
18 except ImportError:
19 PytorchGELUTanh = lambda: nn.GELU(approximate='tanh')
20 from transformers.modeling_utils import PreTrainedModel
21 from transformers.utils import is_flash_attn_2_available, logging
22
23 if is_flash_attn_2_available():
24 from flash_attn import flash_attn_varlen_func
25 else:
26 flash_attn_varlen_func = None
27
28 from transformers.configuration_utils import PretrainedConfig
29
30
31 logger = logging.get_logger(__name__)
32
33
34 class MoonViTConfig(PretrainedConfig):
35 model_type = "moonvit"
36
37 def __init__(
38 self,
39 patch_size: int = 14,
40 init_pos_emb_height: int = 64,
41 init_pos_emb_width: int = 64,
42 num_attention_heads: int = 16,
43 num_hidden_layers: int = 27,
44 hidden_size: int = 1152,
45 intermediate_size: int = 4304,
46 merge_kernel_size: tuple[int, int] = (2, 2),
47 **kwargs,
48 ):
49 super().__init__(**kwargs)
50 self.patch_size = patch_size
51 # Positional embedding config
52 self.init_pos_emb_height = init_pos_emb_height
53 self.init_pos_emb_width = init_pos_emb_width
54 # Transformer config
55 self.num_hidden_layers = num_hidden_layers
56 self.num_attention_heads = num_attention_heads
57 self.hidden_size = hidden_size
58 self.intermediate_size = intermediate_size
59 # Patch merger config
60 self.merge_kernel_size = merge_kernel_size
61
62
63 def multihead_attention(
64 q: torch.Tensor,
65 k: torch.Tensor,
66 v: torch.Tensor,
67 q_cu_seqlens: Optional[torch.Tensor] = None,
68 k_cu_seqlens: Optional[torch.Tensor] = None,
69 ):
70 """Multi-head attention using flash attention 2.
71
72 Args:
73 q, k, v: tensor of shape (batch_size, seqlen, num_heads, head_dim),
74 or (tot_seqlens, num_heads, head_dim) if packing.
75 q_cu_seqlens (torch.Tensor): cumulative sequence lengths of q.
76 The first element should be 0 and the last element should be q.shape[0].
77 k_cu_seqlens (torch.Tensor): cumulative sequence lengths of k.
78 The first element should be 0 and the last element should be k.shape[0].
79
80 Returns:
81 output: shape (batch_size, seqlen, dim) or (tot_seqlens, dim) if packing,
82 where dim = num_heads * head_dim
83 """
84 if flash_attn_varlen_func is None:
85 logger.warning_once(
86 "flash_attn is not available for MoonViT; falling back to sdpa attention."
87 )
88 return sdpa_attention(
89 q,
90 k,
91 v,
92 q_cu_seqlens=q_cu_seqlens,
93 k_cu_seqlens=k_cu_seqlens,
94 )
95
96 # Unified format legal check
97 assert q.dim() == k.dim() == v.dim() == 3, "q, k, v must have 3 dims"
98 assert q_cu_seqlens[-1] == q.shape[0], "q_cu_seqlens must sum to q.shape[0]"
99 assert (
100 k_cu_seqlens[-1] == k.shape[0] == v.shape[0]
101 ), "k_cu_seqlens must sum to k.shape[0]"
102 assert q.dtype in [
103 torch.bfloat16,
104 torch.float16,
105 ], f"unsupported dtype {q.dtype} for multihead attn"
106
107 max_seqlen_q = (q_cu_seqlens[1:] - q_cu_seqlens[:-1]).max().item()
108 max_seqlen_k = (k_cu_seqlens[1:] - k_cu_seqlens[:-1]).max().item()
109 attn_out = flash_attn_varlen_func(
110 q,
111 k,
112 v,
113 q_cu_seqlens,
114 k_cu_seqlens,
115 max_seqlen_q,
116 max_seqlen_k,
117 causal=False,
118 )
119 attn_out = attn_out.flatten(start_dim=-2)
120
121 return attn_out
122
123
124 def sdpa_attention(
125 q: torch.Tensor,
126 k: torch.Tensor,
127 v: torch.Tensor,
128 q_cu_seqlens: Optional[torch.Tensor] = None,
129 k_cu_seqlens: Optional[torch.Tensor] = None,
130 ) -> torch.Tensor:
131 """SDPA attention.
132
133 Args:
134 q, k, v: tensor of shape (batch_size, seqlen, num_heads, head_dim),
135 or (tot_seqlens, num_heads, head_dim) if packing.
136 """
137 seq_length = q.shape[0]
138 attention_mask = torch.zeros(
139 [1, seq_length, seq_length], device=q.device, dtype=torch.bool
140 )
141 for i in range(1, len(q_cu_seqlens)):
142 attention_mask[
143 ...,
144 q_cu_seqlens[i - 1] : q_cu_seqlens[i],
145 q_cu_seqlens[i - 1] : q_cu_seqlens[i],
146 ] = True
147 q = q.transpose(0, 1)
148 k = k.transpose(0, 1)
149 v = v.transpose(0, 1)
150 attn_output = F.scaled_dot_product_attention(q, k, v, attention_mask, dropout_p=0.0)
151 attn_output = attn_output.transpose(0, 1)
152 attn_output = attn_output.reshape(seq_length, -1)
153 return attn_output
154
155
156 def eager_attention(
157 q: torch.Tensor,
158 k: torch.Tensor,
159 v: torch.Tensor,
160 q_cu_seqlens: Optional[torch.Tensor] = None,
161 k_cu_seqlens: Optional[torch.Tensor] = None,
162 ) -> torch.Tensor:
163 seq_length = q.shape[0]
164 attention_mask = torch.zeros(
165 [1, seq_length, seq_length], device=q.device, dtype=torch.bool
166 )
167 for i in range(1, len(q_cu_seqlens)):
168 attention_mask[
169 ...,
170 q_cu_seqlens[i - 1] : q_cu_seqlens[i],
171 q_cu_seqlens[i - 1] : q_cu_seqlens[i],
172 ] = True
173 q = q.transpose(0, 1)
174 k = k.transpose(0, 1)
175 v = v.transpose(0, 1)
176
177 attn_weight = q @ k.transpose(-2, -1) / math.sqrt(q.shape[-1])
178 attn_weight += attention_mask
179 attn_weight = torch.softmax(attn_weight, dim=-1, dtype=torch.float32).to(q.dtype)
180
181 attn_output = attn_weight @ v
182 attn_output = attn_output.transpose(0, 1)
183 attn_output = attn_output.reshape(seq_length, -1)
184 return attn_output
185
186
187 VL_VISION_ATTENTION_FUNCTIONS = {
188 "flash_attention_2": multihead_attention,
189 "sdpa": sdpa_attention,
190 "eager": eager_attention,
191 }
192
193
194 def _apply_rope_input_validation(x, freqs_cis):
195 assert x.ndim == freqs_cis.ndim + 1, (x.shape, freqs_cis.shape)
196 assert x.shape[:-2] == freqs_cis.shape[:-1], (x.shape, freqs_cis.shape)
197 assert x.shape[-1] == 2 * freqs_cis.shape[-1], (x.shape, freqs_cis.shape)
198 assert freqs_cis.dtype == torch.complex64, freqs_cis.dtype
199
200
201 def apply_rope(
202 xq: torch.Tensor, xk: torch.Tensor, freqs_cis: torch.Tensor
203 ) -> tuple[torch.Tensor, torch.Tensor]:
204 """
205 Args: (The leading dimensions of all inputs should be the same)
206 xq: query, tensor of shape (..., num_heads, head_dim)
207 xk: key, tensor of shape (..., num_heads, head_dim)
208 freqs_cis: tensor of shape (..., head_dim/2), dtype=torch.complex64. It contains the precomputed cis(freqs) for each position in the 2D grid.
209 Returns:
210 xq_out, xk_out: tensors of shape (..., num_heads, head_dim)
211 """
212 _apply_rope_input_validation(xq, freqs_cis)
213 _apply_rope_input_validation(xk, freqs_cis)
214
215 freqs_cis = freqs_cis.unsqueeze(-2) # ..., 1, head_dim/2
216 # ..., num_heads, head_dim/2
217 xq_ = torch.view_as_complex(xq.float().view(*xq.shape[:-1], -1, 2))
218 xk_ = torch.view_as_complex(xk.float().view(*xq.shape[:-1], -1, 2))
219 xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(-2) # ..., num_heads, head_dim
220 xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(-2) # ..., num_heads, head_dim
221 return xq_out.type_as(xq), xk_out.type_as(xk)
222
223
224 class Learnable2DInterpPosEmb(nn.Module):
225 def __init__(
226 self, height: int, width: int, dim: int, interpolation_mode: str = "bicubic"
227 ) -> None:
228 super().__init__()
229 self.height = height
230 self.width = width
231 self.interpolation_mode = interpolation_mode
232 self.weight = nn.Parameter(torch.empty(height, width, dim))
233 self.reset_parameters()
234
235 def reset_parameters(self):
236 nn.init.normal_(self.weight)
237
238 def forward(self, x: torch.Tensor, grid_hws: torch.Tensor) -> torch.Tensor:
239 pos_embs = []
240 for shape in grid_hws.tolist():
241 if shape == self.weight.shape[:-1]:
242 pos_embs.append(self.weight.flatten(end_dim=1))
243 else:
244 pos_embs.append(
245 F.interpolate(
246 self.weight.permute((2, 0, 1)).unsqueeze(0),
247 size=shape,
248 mode=self.interpolation_mode,
249 )
250 .squeeze(0)
251 .permute((1, 2, 0))
252 .flatten(end_dim=1)
253 )
254 out = x + torch.cat(pos_embs)
255 return out
256
257
258 class MoonVisionPatchEmbed(nn.Module):
259
260 def __init__(
261 self,
262 out_dim: int,
263 in_dim: int = 3,
264 patch_size: Union[int, Tuple[int, int]] = (14, 14),
265 pos_emb_height: int = 14,
266 pos_emb_width: int = 14,
267 ):
268 super().__init__()
269 assert isinstance(
270 patch_size, (int, Sequence)
271 ), f"Invalid patch_size type: {type(patch_size)}"
272 if isinstance(patch_size, int):
273 patch_size = (patch_size, patch_size)
274 assert (
275 len(patch_size) == 2
276 ), f"Expected patch_size to be a tuple of 2, got {patch_size}"
277 self.patch_size = patch_size
278
279 self.proj = nn.Conv2d(
280 in_dim, out_dim, kernel_size=patch_size, stride=patch_size
281 )
282
283 self.pos_emb = Learnable2DInterpPosEmb(
284 height=pos_emb_height, width=pos_emb_width, dim=out_dim
285 )
286
287 def forward(self, x: torch.Tensor, grid_hws: torch.Tensor) -> torch.Tensor:
288 """
289 Args:
290 x (L, Channels): input tensor
291 grid_hws (N, 2): grid height and width
292
293 Returns:
294 (L, Cout) tensor
295 """
296 x = self.proj(x).view(x.size(0), -1)
297 # apply positional embedding
298 x = self.pos_emb(x, grid_hws)
299 return x
300
301
302 class Rope2DPosEmb(nn.Module):
303 """2D rotary position embedding with multi-resolution support.
304
305 This class is intended to be used in the following way:
306 1. Before training, create an instance of Rope2DPosEmb. This instance will hold the precomputed cis.
307 2. Before each forward pass, call `get_freqs_cis_by_*` to get the `freqs_cis` tensor for this iteration.
308 3. During the forward pass, pass the `freqs_cis` tensor to each attention layer, and call `apply` just before each attention operation.
309 The rope is shared across all attention layers and all heads.
310
311 Refs:
312 - RoFormer: https://arxiv.org/abs/2104.09864
313 - VisionLLaMA: https://arxiv.org/abs/2403.00522
314 - https://github.com/Meituan-AutoML/VisionLLaMA/blob/main/dit/models.py
315
316 Args:
317 dim (int): usually the multi-head attention dimension, should be divisible by 4 (TODO: relax this constraint if needed)
318 max_height (int): the maximum height of the 2D grid
319 max_width (int): the maximum width of the 2D grid
320 theta_base (float): the base of the theta
321 device (str): the device to store the precomputed cis
322 """
323
324 def __init__(self, dim: int, max_height: int, max_width: int, theta_base=10000):
325 super().__init__()
326 self.dim = dim
327 assert self.dim % 4 == 0, "dim must be divisible by 4"
328 self.max_height = max_height
329 self.max_width = max_width
330 self.theta_base = theta_base
331
332 self.freqs_cis = None
333
334 def extra_repr(self):
335 return f"dim={self.dim}, max_height={self.max_height}, max_width={self.max_width}, theta_base={self.theta_base}"
336
337 def _precompute_freqs_cis(self, device: torch.device) -> torch.Tensor:
338 """Calculate the cis(freqs) for each position in the 2D grid.
339
340 Return: complex tensor of shape (max_height, max_width, dim//2) and value:
341 height axis: ret[h, w, 2*i] = cis(h * theta_base**(-4*i/dim))
342 weight axis: ret[h, w, 2*i+1] = cis(w * theta_base**(-4*i/dim)) with (i in [0, dim//4))
343 note: `cis` is a mathematical notation defined by cis x = cos x + i sin x,
344 """
345 N = self.max_height * self.max_width
346 flat_pos = torch.arange(0, N).float().to(device)
347 x_pos = flat_pos % self.max_width
348 y_pos = flat_pos // self.max_width
349 dim_range = (
350 torch.arange(0, self.dim, 4)[: (self.dim // 4)].float().to(device)
351 ) # C/4
352 freqs = 1.0 / (self.theta_base ** (dim_range / self.dim))
353 x_freqs = torch.outer(x_pos, freqs).float() # N, C/4
354 y_freqs = torch.outer(y_pos, freqs).float() # N, C/4
355 x_cis = torch.polar(torch.ones_like(x_freqs), x_freqs) # N, C/4
356 y_cis = torch.polar(torch.ones_like(y_freqs), y_freqs) # N, C/4
357 # N, C/4, 2
358 freqs_cis = torch.cat(
359 [x_cis.unsqueeze(dim=-1), y_cis.unsqueeze(dim=-1)], dim=-1
360 )
361 # max_height, max_width, C/2
362 freqs_cis = freqs_cis.reshape(self.max_height, self.max_width, -1)
363 return freqs_cis
364
365 def get_freqs_cis(self, grid_hws: torch.Tensor) -> torch.Tensor:
366 """
367 Args:
368 grid_hws (torch.Tensor): grid height and width
369
370 Returns:
371 freqs_cis: tensor of shape (sum(t * height * width), dim//2)
372 """
373 if self.freqs_cis is None:
374 self.freqs_cis = self._precompute_freqs_cis(grid_hws.device)
375
376 shapes = grid_hws.tolist()
377 assert all(
378 1 <= h <= self.max_height and 1 <= w <= self.max_width for h, w in shapes
379 ), (
380 shapes,
381 self.max_height,
382 self.max_width,
383 )
384 freqs_cis = torch.cat(
385 [self.freqs_cis[:h, :w].reshape(-1, self.dim // 2) for h, w in shapes],
386 dim=0,
387 )
388 return freqs_cis
389
390
391 class MLP2(nn.Module):
392 """
393 Args:
394 dims: [in_dim, hidden_dim, out_dim]
395 bias: whether to use bias in linear layer.
396 """
397
398 def __init__(self, dims: list[int], activation, bias=True):
399 super().__init__()
400 assert len(dims) == 3
401 self.fc0 = nn.Linear(dims[0], dims[1], bias=bias)
402 self.fc1 = nn.Linear(dims[1], dims[2], bias=bias)
403 self.activation = activation
404 for m in [self.fc0, self.fc1]:
405 nn.init.trunc_normal_(m.weight, std=math.sqrt(2 / m.in_features))
406 if m.bias is not None:
407 nn.init.zeros_(m.bias)
408
409 def forward(self, x: torch.Tensor) -> torch.Tensor:
410 x = self.fc0(x)
411 x = self.activation(x)
412 return self.fc1(x)
413
414
415 class MoonVitEncoderLayer(nn.Module):
416
417 def __init__(
418 self,
419 num_heads: int,
420 hidden_dim: int,
421 mlp_dim: int,
422 *,
423 attn_implementation: str = "eager",
424 activation=F.gelu,
425 attn_bias: bool = False,
426 ):
427 super().__init__()
428 self.num_heads = num_heads
429 self.hidden_dim = hidden_dim
430 self.hidden_size_per_attention_head = self.hidden_dim // self.num_heads
431 self.attn_implementation = attn_implementation
432
433 self.norm0 = nn.LayerNorm(hidden_dim)
434 self.norm1 = nn.LayerNorm(hidden_dim)
435 self.mlp = MLP2([hidden_dim, mlp_dim, hidden_dim], activation)
436 self.wqkv = nn.Linear(hidden_dim, hidden_dim * 3, bias=attn_bias)
437 self.wo = nn.Linear(hidden_dim, hidden_dim, bias=attn_bias)
438
439 def attention_qkvpacked(
440 self,
441 x: torch.Tensor,
442 cu_seqlens: torch.Tensor,
443 rope_freqs_cis: Optional[torch.Tensor] = None,
444 ):
445 """
446 Args:
447 x (torch.Tensor): (batch_size, seqlen, hidden_dim)
448 cu_seqlens (torch.Tensor):
449 """
450 xqkv = self.wqkv(x)
451
452 qkv_shape = xqkv.size()[:-1] + (
453 3,
454 self.num_heads,
455 self.hidden_size_per_attention_head,
456 )
457 # xqkv: (batch_size, seqlen, 3, nheads, headdim)
458 xqkv = xqkv.view(*qkv_shape)
459 xq, xk, xv = torch.unbind(xqkv, dim=-3)
460
461 xq, xk = apply_rope(xq, xk, rope_freqs_cis)
462
463 attn_func = VL_VISION_ATTENTION_FUNCTIONS[self.attn_implementation]
464 attn_out = attn_func(
465 xq, xk, xv, q_cu_seqlens=cu_seqlens, k_cu_seqlens=cu_seqlens
466 )
467
468 attn_out = self.wo(attn_out)
469 return attn_out
470
471 def forward(
472 self,
473 hidden_states: torch.Tensor,
474 cu_seqlens: torch.Tensor,
475 rope_freqs_cis: Union[torch.Tensor, None] = None,
476 ) -> torch.Tensor:
477 """
478 Args:
479 hidden_states: non-packed (B, N, D) or packed (L, D). if non-packed, seqlens should be None, if packed, seqlens should be set
480
481 Returns:
482 output: same shape of input, non-packed (B, N, D) for non-packed input, (L, D) for packed input
483 """
484 residual = hidden_states
485 hidden_states = self.norm0(hidden_states)
486 attn_out = self.attention_qkvpacked(
487 hidden_states, cu_seqlens, rope_freqs_cis=rope_freqs_cis
488 )
489 hidden_states = residual + attn_out
490
491 residual = hidden_states
492 hidden_states = self.mlp(self.norm1(hidden_states))
493 hidden_states = residual + hidden_states
494 return hidden_states
495
496
497 class MoonVitEncoder(nn.Module):
498
499 def __init__(
500 self,
501 hidden_dim: int,
502 num_layers: int,
503 block_cfg: dict,
504 ) -> None:
505 super().__init__()
506
507 self.rope_2d = Rope2DPosEmb(
508 block_cfg["hidden_dim"] // block_cfg["num_heads"], 512, 512
509 )
510 self.blocks = nn.ModuleList(
511 [MoonVitEncoderLayer(**block_cfg) for _ in range(num_layers)]
512 )
513 self.final_layernorm = nn.LayerNorm(hidden_dim)
514
515 def forward(
516 self, hidden_states: torch.Tensor, grid_hws: torch.Tensor
517 ) -> torch.Tensor:
518 rope_freqs_cis = self.rope_2d.get_freqs_cis(grid_hws=grid_hws)
519
520 lengths = torch.cat(
521 (
522 torch.zeros(1, device=hidden_states.device, dtype=grid_hws.dtype),
523 grid_hws[:, 0] * grid_hws[:, 1],
524 )
525 )
526 cu_seqlens = lengths.cumsum(dim=0, dtype=torch.int32)
527
528 for _, block in enumerate(self.blocks):
529 hidden_states = block(
530 hidden_states, cu_seqlens, rope_freqs_cis=rope_freqs_cis
531 )
532
533 hidden_states = self.final_layernorm(hidden_states)
534
535 return hidden_states
536
537
538 def patch_merger(
539 x: torch.Tensor,
540 grid_hws: torch.Tensor,
541 merge_kernel_size: list[int, int] = (2, 2),
542 ) -> List[torch.Tensor]:
543 d_model = x.size(-1)
544
545 outputs = []
546 pre_sum = 0
547 for x_shape in grid_hws.tolist():
548 height, width = x_shape[0], x_shape[1]
549 # Get the current sequence
550 seq = x[pre_sum : pre_sum + height * width]
551 # Reshape along self.merge_kernel_size and concat to the last dimension
552 kernel_height, kernel_width = merge_kernel_size
553 new_height, new_width = height // kernel_height, width // kernel_width
554 reshaped_seq = seq.view(
555 new_height, kernel_height, new_width, kernel_width, d_model
556 )
557 reshaped_seq = reshaped_seq.permute(0, 2, 1, 3, 4).contiguous()
558 padded_seq = reshaped_seq.view(
559 new_height * new_width, -1
560 )
561 outputs.append(padded_seq)
562 pre_sum += height * width
563
564 return outputs
565
566
567 class MoonVitPretrainedModel(PreTrainedModel):
568 config_class = MoonViTConfig
569 model_type = "moonvit"
570 _no_split_modules = ["PackingTransformer"]
571 _supports_flash_attn_2 = True
572 _supports_sdpa = True
573
574 def __init__(self, config: MoonViTConfig, *inputs, **kwargs):
575 super().__init__(config, *inputs, **kwargs)
576 config = deepcopy(config)
577 self.merge_kernel_size = config.merge_kernel_size
578 self.patch_size = config.patch_size
579 self.patch_embed = MoonVisionPatchEmbed(
580 out_dim=config.hidden_size,
581 patch_size=config.patch_size,
582 pos_emb_height=config.init_pos_emb_height,
583 pos_emb_width=config.init_pos_emb_width,
584 )
585
586 self.encoder = MoonVitEncoder(
587 hidden_dim=config.hidden_size,
588 num_layers=config.num_hidden_layers,
589 block_cfg={
590 "num_heads": config.num_attention_heads,
591 "hidden_dim": config.hidden_size,
592 "mlp_dim": config.intermediate_size,
593 "activation": PytorchGELUTanh(),
594 "attn_bias": True,
595 "attn_implementation": config._attn_implementation,
596 },
597 )
598
599 def forward(
600 self, pixel_values: torch.Tensor, grid_hws: torch.Tensor
601 ) -> torch.Tensor:
602 """
603 Args:
604 pixel_values (torch.Tensor): The input pixel values.
605 grid_hws (torch.Tensor): The grid height and width.
606
607 Returns:
608 torch.Tensor: The output tokens.
609 """
610 hidden_states = self.patch_embed(pixel_values, grid_hws)
611 hidden_states = self.encoder(hidden_states, grid_hws)
612 hidden_states = patch_merger(
613 hidden_states, grid_hws, merge_kernel_size=self.merge_kernel_size
614 )
615 return hidden_states
616