de-thorsten-1p57m/runtime-kernels.md
7.6 KB · 324 lines · markdown Raw
1 # Root A Runtime Kernels
2
3 This package is meant to be loaded by a small standalone runtime, not by
4 PyTorch. The runtime should read `manifest.json`, mmap or read
5 `weights.fp16.bin`, and dispatch tensors by `offset_bytes`.
6
7 ## Input Boundary
8
9 Current supported input is a Piper phoneme ID sequence. Arbitrary text is not
10 self-contained yet because Piper/eSpeak phonemization is still external.
11
12 ## Duration Student
13
14 Config:
15
16 ```json
17 {
18 "architecture": "duration_conv",
19 "vocab_size": 142,
20 "hidden": 64,
21 "depth": 3,
22 "kernel_size": 5,
23 "max_tokens": 759,
24 "max_duration": 80,
25 "target_duration_scale": 1.0,
26 "pause_preserve_scale": 0.0,
27 "pause_preserve_window": 0,
28 "pause_token_ids": [
29 4,
30 8,
31 10,
32 11,
33 12,
34 13
35 ],
36 "long_preserve_threshold": 0.0,
37 "long_preserve_scale": 0.0,
38 "duration_vocab_size_forced": 0,
39 "duration_oov_id": 59
40 }
41 ```
42
43 Inputs:
44
45 - `ids`: int token IDs, shape `[tokens]`
46 - `mask`: implicit all-valid for a single sequence
47
48 Ops:
49
50 - embedding lookup
51 - position and length hints as defined by the checkpoint config
52 - 1x1 Conv1d
53 - residual masked Conv1d blocks with SiLU
54 - 1x1 Conv1d output
55 - `duration = round(exp(log_duration) * length_scale)`, clamped to the
56 checkpoint's `max_duration`
57
58 ## Acoustic Student
59
60 Config:
61
62 ```json
63 {
64 "architecture": "token_context",
65 "vocab_size": 142,
66 "hidden": 96,
67 "depth": 4,
68 "kernel_size": 5,
69 "out_channels": 192,
70 "token_depth": 3
71 }
72 ```
73
74 Inputs:
75
76 - `ids`: int token IDs, shape `[tokens]`
77 - `durations`: int frame counts, shape `[tokens]`
78
79 Ops:
80
81 - token-context student: token embedding, position/duration hints, token
82 Conv1d blocks, and duration-conditioned frame expansion
83 - repeat token states by duration
84 - frame hints and frame Conv1d blocks
85 - optional calibrated wrapper and output adapter when the config says
86 `architecture=calibrated`
87 - supported current adapter: `depthwise_lowrank`, which applies a small
88 decoder-facing correction without changing the `[192, frames]` latent
89 contract
90 - output Piper generator latent, shape `[192, frames]`
91
92 ## Decoder Student
93
94 Config:
95
96 ```json
97 {
98 "in_channels": 192,
99 "source_in_channels": 192,
100 "channels": [
101 160,
102 80,
103 40,
104 20
105 ],
106 "res_layers": 1,
107 "variant": "piperlite",
108 "lrc_code_dim": 40,
109 "lrc_encoder_hidden": 64,
110 "rank_ratio": 0.5,
111 "activation": "leaky_relu",
112 "stage_affine": false,
113 "factorized_pre_rank": 0,
114 "piper_res_factor_rank_ratio": 0.0,
115 "res_bank_scale_mode": "kept",
116 "stage0_branches": [
117 0,
118 1,
119 2
120 ],
121 "stage1_branches": [
122 0,
123 1,
124 2
125 ],
126 "stage2_branches": [
127 0,
128 1,
129 2
130 ],
131 "stage3_branches": [
132 0,
133 1,
134 2
135 ],
136 "fsd_dim": 72,
137 "fsd_blocks": 5,
138 "fsd_film_rank": 12,
139 "fsd_head_rank": 48,
140 "wavehax_channels": 32,
141 "wavehax_blocks": 8,
142 "wavehax_mult_channels": 2,
143 "wavehax_kernel_freq": 7,
144 "wavehax_kernel_time": 7,
145 "wavehax_f0_channel": 80,
146 "wavehax_voiced_channel": 81,
147 "wavehax_sample_rate": 24000,
148 "wavehax_prior_power": 0.1,
149 "wavehax_prior_noise": 0.01,
150 "whx_channels": 32,
151 "whx_hidden": 64,
152 "whx_blocks": 8,
153 "whx_n_fft": 512,
154 "whx_f0_channel": 80,
155 "whx_voiced_channel": 81,
156 "whx_sample_rate": 24000,
157 "hift_width": 224,
158 "hift_gen_channels": 192,
159 "hift_style_dim": 64,
160 "hift_body_blocks": 3,
161 "hift_f0_channel": 80,
162 "hift_voiced_channel": 81,
163 "hift_sample_rate": 24000,
164 "stage_projection_bottlenecks": [],
165 "teacher_init_checkpoint": null,
166 "teacher_init_method": "first",
167 "teacher_init": null,
168 "init_decoder_checkpoint": "artifacts/voices/de_DE/decoder/decoder-student.pt",
169 "decoder_init": {
170 "checkpoint": "artifacts/voices/de_DE/decoder/decoder-student.pt",
171 "decoder_parameters": 730721,
172 "matched_config": {
173 "in_channels": 192,
174 "channels": [
175 160,
176 80,
177 40,
178 20
179 ],
180 "res_layers": 1,
181 "variant": "piperlite",
182 "rank_ratio": 0.5,
183 "activation": "leaky_relu",
184 "stage_affine": false,
185 "factorized_pre_rank": 0,
186 "piper_res_factor_rank_ratio": 0.0,
187 "res_bank_scale_mode": "kept",
188 "stage0_branches": [
189 0,
190 1,
191 2
192 ],
193 "stage1_branches": [
194 0,
195 1,
196 2
197 ],
198 "stage2_branches": [
199 0,
200 1,
201 2
202 ],
203 "stage3_branches": [
204 0,
205 1,
206 2
207 ],
208 "fsd_dim": 72,
209 "fsd_blocks": 5,
210 "fsd_film_rank": 12,
211 "fsd_head_rank": 48,
212 "wavehax_channels": 32,
213 "wavehax_blocks": 8,
214 "wavehax_mult_channels": 2,
215 "wavehax_kernel_freq": 7,
216 "wavehax_kernel_time": 7,
217 "wavehax_f0_channel": 80,
218 "wavehax_voiced_channel": 81,
219 "wavehax_sample_rate": 24000,
220 "wavehax_prior_power": 0.1,
221 "wavehax_prior_noise": 0.01,
222 "whx_channels": 32,
223 "whx_hidden": 64,
224 "whx_blocks": 8,
225 "whx_n_fft": 512,
226 "whx_f0_channel": 80,
227 "whx_voiced_channel": 81,
228 "whx_sample_rate": 24000,
229 "hift_width": 224,
230 "hift_gen_channels": 192,
231 "hift_style_dim": 64,
232 "hift_body_blocks": 3,
233 "hift_f0_channel": 80,
234 "hift_voiced_channel": 81,
235 "hift_sample_rate": 24000,
236 "stage_projection_bottlenecks": [],
237 "post_filter_channels": 0,
238 "post_filter_layers": 0,
239 "post_filter_kernel": 9,
240 "post_filter_scale": 0.25,
241 "pre_tanh_repair_channels": 0,
242 "pre_tanh_repair_layers": 0,
243 "pre_tanh_repair_kernel": 7,
244 "pre_tanh_repair_scale": 0.15,
245 "istft_n_fft": 512,
246 "istft_tail_head_rank": 20
247 },
248 "allowed_post_filter_mismatches": [],
249 "allowed_pre_tanh_repair_mismatches": [],
250 "allowed_activation_mismatches": [],
251 "missing_keys": [],
252 "unexpected_keys": []
253 },
254 "post_filter_channels": 0,
255 "post_filter_layers": 0,
256 "post_filter_kernel": 9,
257 "post_filter_scale": 0.25,
258 "pre_tanh_repair_channels": 0,
259 "pre_tanh_repair_layers": 0,
260 "pre_tanh_repair_kernel": 7,
261 "pre_tanh_repair_scale": 0.15,
262 "istft_n_fft": 512,
263 "istft_tail_head_rank": 20,
264 "ap_amplitude_weight": 0.0,
265 "ap_phase_weight": 0.0,
266 "ap_complex_weight": 0.0,
267 "spectral_head_init": "default",
268 "spectral_head_init_scale": 0.001,
269 "ap_amp_init_bias": -6.0,
270 "ap_phase_real_init_bias": 1.0,
271 "spectral_head_init_summary": null,
272 "feature_exact_weight": 0.0,
273 "feature_exact_keys": [
274 "pre",
275 "up0"
276 ],
277 "feature_exact_teacher_init": null,
278 "signature_pack_dir": "artifacts/voices/de_DE/signatures",
279 "exact_feature_pack_dir": null,
280 "signature_hint_weight": 0.05,
281 "signature_temporal_weight": 0.4,
282 "signature_phase_weight": 0.0,
283 "signature_phase_bins": 0,
284 "signature_keys": [
285 "stage0_mix",
286 "stage1_mix",
287 "stage2_mix",
288 "pre_tanh",
289 "audio"
290 ],
291 "bottleneck_code_checkpoint": null,
292 "bottleneck_code_weight": 0.0,
293 "bottleneck_code_keys": [],
294 "bottleneck_code_training_only_parameters": 0,
295 "echo_tail_weight": 0.0,
296 "echo_tail_min_ms": 8.0,
297 "echo_tail_max_ms": 40.0,
298 "echo_tail_lags": 9,
299 "echo_tail_margin": 0.02,
300 "paired_acoustic_residual_weight": 0.0,
301 "paired_acoustic_residual_max_scale": 0.25,
302 "lrc_pred_code_mix_prob": 0.0,
303 "lrc_pred_code_checkpoint": null,
304 "lrc_pred_code_residual_prob": 0.0,
305 "lrc_pred_code_residual_max_scale": 0.25,
306 "lrc_pred_code_cache": null
307 }
308 ```
309
310 Inputs:
311
312 - `latent`: float tensor, shape `[192, frames]`
313
314 Ops:
315
316 - PiperLite decoder blocks defined by the checkpoint config
317 - Conv1d same-padding pre/post projections
318 - compact upsampling stages with LeakyReLU for the current A91 branch
319 - low-rank/factorized residual projections when `rank_ratio` is present
320 - tanh waveform output
321
322 The decoder hop is `256` samples per latent frame and the output sample rate is
323 `22050` Hz.
324