Ref2VA/audio_vae/dac_bigvgan.py
6.9 KB · 207 lines · python Raw
1 # SPDX-License-Identifier: MIT
2 # Copyright (c) 2024 NVIDIA CORPORATION.
3 # Licensed under the MIT license.
4
5 # Adapted from https://github.com/jik876/hifi-gan under the MIT license.
6
7 from .dac_activations import SnakeBeta
8
9 import torch
10 import torch.nn as nn
11 from torch.nn import Conv1d, ConvTranspose1d
12 from torch.nn.utils.parametrizations import weight_norm
13
14 from .dac_utils import init_weights, get_padding
15 from .dac_alias_free_act import Activation1d
16
17
18 class AttrDict(dict):
19 def __init__(self, *args, **kwargs):
20 super(AttrDict, self).__init__(*args, **kwargs)
21 self.__dict__ = self
22
23
24 class AMPBlock1(torch.nn.Module):
25 """
26 AMPBlock applies SnakeBeta activation functions with trainable parameters that control periodicity, defined for each layer.
27 AMPBlock1 has additional self.convs2 that contains additional Conv1d layers with a fixed dilation=1 followed by each layer in self.convs1
28
29 Args:
30 h (AttrDict): Hyperparameters.
31 channels (int): Number of convolution channels.
32 kernel_size (int): Size of the convolution kernel. Default is 3.
33 dilation (tuple): Dilation rates for the convolutions. Each dilation layer has two convolutions. Default is (1, 3, 5).
34 activation (str): Activation function type. Must be 'snakebeta'.
35 """
36
37 def __init__(
38 self,
39 h: AttrDict,
40 channels: int,
41 kernel_size: int = 3,
42 dilation: tuple = (1, 3, 5),
43 activation: str = None,
44 ):
45 super().__init__()
46
47 self.h = h
48
49 self.convs1 = nn.ModuleList(
50 [
51 weight_norm(
52 Conv1d(
53 channels,
54 channels,
55 kernel_size,
56 stride=1,
57 dilation=d,
58 padding=get_padding(kernel_size, d),
59 )
60 )
61 for d in dilation
62 ]
63 )
64 self.convs1.apply(init_weights)
65
66 self.convs2 = nn.ModuleList(
67 [
68 weight_norm(
69 Conv1d(
70 channels,
71 channels,
72 kernel_size,
73 stride=1,
74 dilation=1,
75 padding=get_padding(kernel_size, 1),
76 )
77 )
78 for _ in range(len(dilation))
79 ]
80 )
81 self.convs2.apply(init_weights)
82
83 self.num_layers = len(self.convs1) + len(self.convs2) # Total number of conv layers
84
85 if activation == "snakebeta":
86 self.activations = nn.ModuleList(
87 [
88 Activation1d(activation=SnakeBeta(channels, alpha_logscale=h.snake_logscale))
89 for _ in range(self.num_layers)
90 ]
91 )
92 else:
93 raise NotImplementedError(
94 "activation incorrectly specified. check the config file and look for 'activation'."
95 )
96
97 def forward(self, x):
98 acts1, acts2 = self.activations[::2], self.activations[1::2]
99 for c1, c2, a1, a2 in zip(self.convs1, self.convs2, acts1, acts2):
100 xt = a1(x)
101 xt = c1(xt)
102 xt = a2(xt)
103 xt = c2(xt)
104 x = xt + x
105
106 return x
107
108
109 class BigVGAN(torch.nn.Module):
110 """
111 BigVGAN is a neural vocoder model that applies anti-aliased periodic activation for residual blocks (resblocks).
112
113 Args:
114 h (AttrDict): Hyperparameters.
115 """
116
117 def __init__(self, h: AttrDict):
118 super().__init__()
119 self.h = h
120
121 self.num_kernels = len(h.resblock_kernel_sizes)
122 self.num_upsamples = len(h.upsample_rates)
123
124 # Pre-conv
125 self.conv_pre = weight_norm(Conv1d(h.num_mels, h.upsample_initial_channel, 7, 1, padding=3))
126
127 # Define which AMPBlock to use. BigVGAN uses AMPBlock1 as default
128 if h.resblock == "1":
129 resblock_class = AMPBlock1
130 else:
131 raise ValueError(f"Incorrect resblock class specified in hyperparameters. Got {h.resblock}")
132
133 # Transposed conv-based upsamplers. does not apply anti-aliasing
134 self.ups = nn.ModuleList()
135 for i, (u, k) in enumerate(zip(h.upsample_rates, h.upsample_kernel_sizes)):
136 self.ups.append(
137 nn.ModuleList(
138 [
139 weight_norm(
140 ConvTranspose1d(
141 h.upsample_initial_channel // (2**i),
142 h.upsample_initial_channel // (2 ** (i + 1)),
143 k,
144 u,
145 padding=(k - u) // 2,
146 )
147 )
148 ]
149 )
150 )
151
152 # Residual blocks using anti-aliased multi-periodicity composition modules (AMP)
153 self.resblocks = nn.ModuleList()
154 for i in range(len(self.ups)):
155 ch = h.upsample_initial_channel // (2 ** (i + 1))
156 for j, (k, d) in enumerate(zip(h.resblock_kernel_sizes, h.resblock_dilation_sizes)):
157 self.resblocks.append(resblock_class(h, ch, k, d, activation=h.activation))
158
159 # Post-conv
160 if h.activation != "snakebeta":
161 raise NotImplementedError(
162 "activation incorrectly specified. check the config file and look for 'activation'."
163 )
164 activation_post = SnakeBeta(ch, alpha_logscale=h.snake_logscale)
165
166 self.activation_post = Activation1d(activation=activation_post)
167
168 # Whether to use bias for the final conv_post. Default to True for backward compatibility
169 self.use_bias_at_final = h.get("use_bias_at_final", True)
170 self.conv_post = weight_norm(Conv1d(ch, 1, 7, 1, padding=3, bias=self.use_bias_at_final))
171
172 # Weight initialization
173 for i in range(len(self.ups)):
174 self.ups[i].apply(init_weights)
175 self.conv_post.apply(init_weights)
176
177 # Final tanh activation. Defaults to True for backward compatibility
178 self.use_tanh_at_final = h.get("use_tanh_at_final", True)
179
180 def forward(self, x):
181 # Pre-conv
182 x = self.conv_pre(x)
183
184 for i in range(self.num_upsamples):
185 # Upsampling
186 for i_up in range(len(self.ups[i])):
187 x = self.ups[i][i_up](x)
188 # AMP blocks
189 xs = None
190 for j in range(self.num_kernels):
191 if xs is None:
192 xs = self.resblocks[i * self.num_kernels + j](x)
193 else:
194 xs += self.resblocks[i * self.num_kernels + j](x)
195 x = xs / self.num_kernels
196
197 # Post-conv
198 x = self.activation_post(x)
199 x = self.conv_post(x)
200 # Final tanh activation
201 if self.use_tanh_at_final:
202 x = torch.tanh(x)
203 else:
204 x = torch.clamp(x, min=-1.0, max=1.0) # Bound the output to [-1, 1]
205
206 return x
207