FL2VA/audio_vae/dac_attn_proj.py
| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | import torch |
| 3 | import torch.nn as nn |
| 4 | import torch.nn.functional as F |
| 5 | from torch.nn.functional import scaled_dot_product_attention |
| 6 | |
| 7 | |
| 8 | class GeGluMlp(nn.Module): |
| 9 | def __init__( |
| 10 | self, |
| 11 | in_features, |
| 12 | hidden_features, |
| 13 | ): |
| 14 | super().__init__() |
| 15 | self.norm = nn.LayerNorm(in_features) |
| 16 | self.act = nn.GELU(approximate="tanh") |
| 17 | self.w0 = nn.Linear(in_features, hidden_features) |
| 18 | self.w1 = nn.Linear(in_features, hidden_features) |
| 19 | self.w2 = nn.Linear(hidden_features, in_features) |
| 20 | |
| 21 | def forward(self, x): |
| 22 | x = self.norm(x) |
| 23 | x = self.act(self.w0(x)) * self.w1(x) |
| 24 | x = self.w2(x) |
| 25 | return x |
| 26 | |
| 27 | |
| 28 | class CausalAttention(nn.Module): |
| 29 | def __init__(self, in_dim, out_dim, num_heads): |
| 30 | super().__init__() |
| 31 | if in_dim > out_dim: |
| 32 | # assert in_dim // num_heads == out_dim |
| 33 | self.head_dim = in_dim // num_heads |
| 34 | self.qkv = nn.Linear(in_dim, in_dim * 3, bias=False) |
| 35 | self.q_bias = nn.Parameter(torch.zeros(in_dim)) |
| 36 | self.v_bias = nn.Parameter(torch.zeros(in_dim)) |
| 37 | self.register_buffer("zero_k_bias", torch.zeros(in_dim)) |
| 38 | else: |
| 39 | # assert out_dim // num_heads == in_dim |
| 40 | self.head_dim = out_dim // num_heads |
| 41 | self.qkv = nn.Linear(in_dim, out_dim * 3, bias=False) |
| 42 | self.q_bias = nn.Parameter(torch.zeros(out_dim)) |
| 43 | self.v_bias = nn.Parameter(torch.zeros(out_dim)) |
| 44 | self.register_buffer("zero_k_bias", torch.zeros(out_dim)) |
| 45 | |
| 46 | self.in_dim = in_dim |
| 47 | self.out_dim = out_dim |
| 48 | self.num_heads = num_heads |
| 49 | self.scale = self.head_dim**-0.5 |
| 50 | self.proj = nn.Linear(out_dim, out_dim) |
| 51 | |
| 52 | def forward(self, x: torch.Tensor) -> torch.Tensor: |
| 53 | B, N, C = x.shape |
| 54 | qkv = F.linear(input=x, weight=self.qkv.weight, bias=torch.cat((self.q_bias, self.zero_k_bias, self.v_bias))) |
| 55 | q, k, v = qkv.reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4).unbind(0) |
| 56 | |
| 57 | x = scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=True) |
| 58 | |
| 59 | if self.in_dim > self.out_dim: |
| 60 | x = torch.mean(x, dim=1) |
| 61 | if self.in_dim // self.num_heads != self.out_dim: |
| 62 | x = nn.functional.adaptive_avg_pool1d(x, self.out_dim) |
| 63 | else: |
| 64 | x = x.transpose(1, 2).reshape(B, N, -1) |
| 65 | x = self.proj(x) |
| 66 | return x |
| 67 | |
| 68 | |
| 69 | class AttnProjection(nn.Module): |
| 70 | def __init__(self, in_dim, out_dim, num_heads, norm_layer=nn.LayerNorm, mlp_ratio=2): |
| 71 | super().__init__() |
| 72 | assert out_dim % in_dim == 0 or in_dim % out_dim == 0 |
| 73 | self.in_dim = in_dim |
| 74 | self.out_dim = out_dim |
| 75 | self.norm1 = norm_layer(in_dim) |
| 76 | self.attn = CausalAttention(in_dim, out_dim, num_heads) |
| 77 | self.proj = nn.Linear(in_dim, out_dim) |
| 78 | self.norm3 = norm_layer(in_dim) |
| 79 | |
| 80 | self.norm2 = norm_layer(out_dim) |
| 81 | hidden_dim = int(out_dim * mlp_ratio) |
| 82 | self.mlp = GeGluMlp(in_features=out_dim, hidden_features=hidden_dim) |
| 83 | # self.mlp = FeedForward(out_dim, out_dim) |
| 84 | |
| 85 | def forward(self, x): |
| 86 | x = self.proj(self.norm3(x)) + self.attn(self.norm1(x)) |
| 87 | x = x + self.mlp(self.norm2(x)) |
| 88 | return x |
| 89 | |