fr-siwis-1p57m/runtime-kernels.md
7.5 KB · 322 lines · markdown Raw
1 # Root A Runtime Kernels
2
3 This package is meant to be loaded by a small standalone runtime, not by
4 PyTorch. The runtime should read `manifest.json`, mmap or read
5 `weights.fp16.bin`, and dispatch tensors by `offset_bytes`.
6
7 ## Input Boundary
8
9 Current supported input is a Piper phoneme ID sequence. Arbitrary text is not
10 self-contained yet because Piper/eSpeak phonemization is still external.
11
12 ## Duration Student
13
14 Config:
15
16 ```json
17 {
18 "architecture": "duration_conv",
19 "vocab_size": 142,
20 "hidden": 64,
21 "depth": 3,
22 "kernel_size": 5,
23 "max_tokens": 525,
24 "max_duration": 80,
25 "target_duration_scale": 1.0,
26 "pause_preserve_scale": 0.0,
27 "pause_preserve_window": 0,
28 "pause_token_ids": [
29 4,
30 8,
31 10,
32 11,
33 12,
34 13
35 ],
36 "long_preserve_threshold": 0.0,
37 "long_preserve_scale": 0.0
38 }
39 ```
40
41 Inputs:
42
43 - `ids`: int token IDs, shape `[tokens]`
44 - `mask`: implicit all-valid for a single sequence
45
46 Ops:
47
48 - embedding lookup
49 - position and length hints as defined by the checkpoint config
50 - 1x1 Conv1d
51 - residual masked Conv1d blocks with SiLU
52 - 1x1 Conv1d output
53 - `duration = round(exp(log_duration) * length_scale)`, clamped to the
54 checkpoint's `max_duration`
55
56 ## Acoustic Student
57
58 Config:
59
60 ```json
61 {
62 "architecture": "token_context",
63 "vocab_size": 142,
64 "hidden": 96,
65 "depth": 4,
66 "kernel_size": 5,
67 "out_channels": 192,
68 "token_depth": 3
69 }
70 ```
71
72 Inputs:
73
74 - `ids`: int token IDs, shape `[tokens]`
75 - `durations`: int frame counts, shape `[tokens]`
76
77 Ops:
78
79 - token-context student: token embedding, position/duration hints, token
80 Conv1d blocks, and duration-conditioned frame expansion
81 - repeat token states by duration
82 - frame hints and frame Conv1d blocks
83 - optional calibrated wrapper and output adapter when the config says
84 `architecture=calibrated`
85 - supported current adapter: `depthwise_lowrank`, which applies a small
86 decoder-facing correction without changing the `[192, frames]` latent
87 contract
88 - output Piper generator latent, shape `[192, frames]`
89
90 ## Decoder Student
91
92 Config:
93
94 ```json
95 {
96 "in_channels": 192,
97 "source_in_channels": 192,
98 "channels": [
99 160,
100 80,
101 40,
102 20
103 ],
104 "res_layers": 1,
105 "variant": "piperlite",
106 "lrc_code_dim": 40,
107 "lrc_encoder_hidden": 64,
108 "rank_ratio": 0.5,
109 "activation": "leaky_relu",
110 "stage_affine": false,
111 "factorized_pre_rank": 0,
112 "piper_res_factor_rank_ratio": 0.0,
113 "res_bank_scale_mode": "kept",
114 "stage0_branches": [
115 0,
116 1,
117 2
118 ],
119 "stage1_branches": [
120 0,
121 1,
122 2
123 ],
124 "stage2_branches": [
125 0,
126 1,
127 2
128 ],
129 "stage3_branches": [
130 0,
131 1,
132 2
133 ],
134 "fsd_dim": 72,
135 "fsd_blocks": 5,
136 "fsd_film_rank": 12,
137 "fsd_head_rank": 48,
138 "wavehax_channels": 32,
139 "wavehax_blocks": 8,
140 "wavehax_mult_channels": 2,
141 "wavehax_kernel_freq": 7,
142 "wavehax_kernel_time": 7,
143 "wavehax_f0_channel": 80,
144 "wavehax_voiced_channel": 81,
145 "wavehax_sample_rate": 24000,
146 "wavehax_prior_power": 0.1,
147 "wavehax_prior_noise": 0.01,
148 "whx_channels": 32,
149 "whx_hidden": 64,
150 "whx_blocks": 8,
151 "whx_n_fft": 512,
152 "whx_f0_channel": 80,
153 "whx_voiced_channel": 81,
154 "whx_sample_rate": 24000,
155 "hift_width": 224,
156 "hift_gen_channels": 192,
157 "hift_style_dim": 64,
158 "hift_body_blocks": 3,
159 "hift_f0_channel": 80,
160 "hift_voiced_channel": 81,
161 "hift_sample_rate": 24000,
162 "stage_projection_bottlenecks": [],
163 "teacher_init_checkpoint": null,
164 "teacher_init_method": "first",
165 "teacher_init": null,
166 "init_decoder_checkpoint": "artifacts/voices/fr_FR/decoder/decoder-student.pt",
167 "decoder_init": {
168 "checkpoint": "artifacts/voices/fr_FR/decoder/decoder-student.pt",
169 "decoder_parameters": 730721,
170 "matched_config": {
171 "in_channels": 192,
172 "channels": [
173 160,
174 80,
175 40,
176 20
177 ],
178 "res_layers": 1,
179 "variant": "piperlite",
180 "rank_ratio": 0.5,
181 "activation": "leaky_relu",
182 "stage_affine": false,
183 "factorized_pre_rank": 0,
184 "piper_res_factor_rank_ratio": 0.0,
185 "res_bank_scale_mode": "kept",
186 "stage0_branches": [
187 0,
188 1,
189 2
190 ],
191 "stage1_branches": [
192 0,
193 1,
194 2
195 ],
196 "stage2_branches": [
197 0,
198 1,
199 2
200 ],
201 "stage3_branches": [
202 0,
203 1,
204 2
205 ],
206 "fsd_dim": 72,
207 "fsd_blocks": 5,
208 "fsd_film_rank": 12,
209 "fsd_head_rank": 48,
210 "wavehax_channels": 32,
211 "wavehax_blocks": 8,
212 "wavehax_mult_channels": 2,
213 "wavehax_kernel_freq": 7,
214 "wavehax_kernel_time": 7,
215 "wavehax_f0_channel": 80,
216 "wavehax_voiced_channel": 81,
217 "wavehax_sample_rate": 24000,
218 "wavehax_prior_power": 0.1,
219 "wavehax_prior_noise": 0.01,
220 "whx_channels": 32,
221 "whx_hidden": 64,
222 "whx_blocks": 8,
223 "whx_n_fft": 512,
224 "whx_f0_channel": 80,
225 "whx_voiced_channel": 81,
226 "whx_sample_rate": 24000,
227 "hift_width": 224,
228 "hift_gen_channels": 192,
229 "hift_style_dim": 64,
230 "hift_body_blocks": 3,
231 "hift_f0_channel": 80,
232 "hift_voiced_channel": 81,
233 "hift_sample_rate": 24000,
234 "stage_projection_bottlenecks": [],
235 "post_filter_channels": 0,
236 "post_filter_layers": 0,
237 "post_filter_kernel": 9,
238 "post_filter_scale": 0.25,
239 "pre_tanh_repair_channels": 0,
240 "pre_tanh_repair_layers": 0,
241 "pre_tanh_repair_kernel": 7,
242 "pre_tanh_repair_scale": 0.15,
243 "istft_n_fft": 512,
244 "istft_tail_head_rank": 20
245 },
246 "allowed_post_filter_mismatches": [],
247 "allowed_pre_tanh_repair_mismatches": [],
248 "allowed_activation_mismatches": [],
249 "missing_keys": [],
250 "unexpected_keys": []
251 },
252 "post_filter_channels": 0,
253 "post_filter_layers": 0,
254 "post_filter_kernel": 9,
255 "post_filter_scale": 0.25,
256 "pre_tanh_repair_channels": 0,
257 "pre_tanh_repair_layers": 0,
258 "pre_tanh_repair_kernel": 7,
259 "pre_tanh_repair_scale": 0.15,
260 "istft_n_fft": 512,
261 "istft_tail_head_rank": 20,
262 "ap_amplitude_weight": 0.0,
263 "ap_phase_weight": 0.0,
264 "ap_complex_weight": 0.0,
265 "spectral_head_init": "default",
266 "spectral_head_init_scale": 0.001,
267 "ap_amp_init_bias": -6.0,
268 "ap_phase_real_init_bias": 1.0,
269 "spectral_head_init_summary": null,
270 "feature_exact_weight": 0.0,
271 "feature_exact_keys": [
272 "pre",
273 "up0"
274 ],
275 "feature_exact_teacher_init": null,
276 "signature_pack_dir": "artifacts/voices/fr_FR/signatures",
277 "exact_feature_pack_dir": null,
278 "signature_hint_weight": 0.05,
279 "signature_temporal_weight": 0.4,
280 "signature_phase_weight": 0.0,
281 "signature_phase_bins": 0,
282 "signature_keys": [
283 "stage0_mix",
284 "stage1_mix",
285 "stage2_mix",
286 "pre_tanh",
287 "audio"
288 ],
289 "bottleneck_code_checkpoint": null,
290 "bottleneck_code_weight": 0.0,
291 "bottleneck_code_keys": [],
292 "bottleneck_code_training_only_parameters": 0,
293 "echo_tail_weight": 0.0,
294 "echo_tail_min_ms": 8.0,
295 "echo_tail_max_ms": 40.0,
296 "echo_tail_lags": 9,
297 "echo_tail_margin": 0.02,
298 "paired_acoustic_residual_weight": 0.0,
299 "paired_acoustic_residual_max_scale": 0.25,
300 "lrc_pred_code_mix_prob": 0.0,
301 "lrc_pred_code_checkpoint": null,
302 "lrc_pred_code_residual_prob": 0.0,
303 "lrc_pred_code_residual_max_scale": 0.25,
304 "lrc_pred_code_cache": null
305 }
306 ```
307
308 Inputs:
309
310 - `latent`: float tensor, shape `[192, frames]`
311
312 Ops:
313
314 - PiperLite decoder blocks defined by the checkpoint config
315 - Conv1d same-padding pre/post projections
316 - compact upsampling stages with LeakyReLU for the current A91 branch
317 - low-rank/factorized residual projections when `rank_ratio` is present
318 - tanh waveform output
319
320 The decoder hop is `256` samples per latent frame and the output sample rate is
321 `22050` Hz.
322