amy-en-1p1m/runtime-kernels.md
6.3 KB · 272 lines · markdown Raw
1 # Root A Runtime Kernels
2
3 This package is meant to be loaded by a small standalone runtime, not by
4 PyTorch. The runtime should read `manifest.json`, mmap or read
5 `weights.fp16.bin`, and dispatch tensors by `offset_bytes`.
6
7 ## Input Boundary
8
9 Current supported input is a Piper phoneme ID sequence. Arbitrary text is not
10 self-contained yet because Piper/eSpeak phonemization is still external.
11
12 ## Duration Student
13
14 Config:
15
16 ```json
17 {
18 "architecture": "duration_conv",
19 "vocab_size": 129,
20 "hidden": 32,
21 "depth": 3,
22 "kernel_size": 5,
23 "max_tokens": 265,
24 "max_duration": 80,
25 "target_duration_scale": 1.0,
26 "pause_preserve_scale": 0.0,
27 "pause_preserve_window": 0,
28 "pause_token_ids": [
29 4,
30 8,
31 10,
32 11,
33 12,
34 13
35 ],
36 "long_preserve_threshold": 0.0,
37 "long_preserve_scale": 0.0
38 }
39 ```
40
41 Inputs:
42
43 - `ids`: int token IDs, shape `[tokens]`
44 - `mask`: implicit all-valid for a single sequence
45
46 Ops:
47
48 - embedding lookup
49 - position and length hints as defined by the checkpoint config
50 - 1x1 Conv1d
51 - residual masked Conv1d blocks with SiLU
52 - 1x1 Conv1d output
53 - `duration = round(exp(log_duration) * length_scale)`, clamped to the
54 checkpoint's `max_duration`
55
56 ## Acoustic Student
57
58 Config:
59
60 ```json
61 {
62 "architecture": "token_context",
63 "vocab_size": 145,
64 "hidden": 64,
65 "depth": 4,
66 "kernel_size": 5,
67 "out_channels": 192,
68 "token_depth": 3
69 }
70 ```
71
72 Inputs:
73
74 - `ids`: int token IDs, shape `[tokens]`
75 - `durations`: int frame counts, shape `[tokens]`
76
77 Ops:
78
79 - token-context student: token embedding, position/duration hints, token
80 Conv1d blocks, and duration-conditioned frame expansion
81 - repeat token states by duration
82 - frame hints and frame Conv1d blocks
83 - optional calibrated wrapper and output adapter when the config says
84 `architecture=calibrated`
85 - supported current adapter: `depthwise_lowrank`, which applies a small
86 decoder-facing correction without changing the `[192, frames]` latent
87 contract
88 - output Piper generator latent, shape `[192, frames]`
89
90 ## Decoder Student
91
92 Config:
93
94 ```json
95 {
96 "in_channels": 192,
97 "source_in_channels": 192,
98 "channels": [
99 160,
100 80,
101 40,
102 20
103 ],
104 "res_layers": 1,
105 "variant": "piperlite",
106 "lrc_code_dim": 40,
107 "lrc_encoder_hidden": 64,
108 "rank_ratio": 0.5,
109 "activation": "leaky_relu",
110 "stage_affine": false,
111 "factorized_pre_rank": 0,
112 "piper_res_factor_rank_ratio": 0.0,
113 "res_bank_scale_mode": "kept",
114 "stage0_branches": [
115 0,
116 1,
117 2
118 ],
119 "stage1_branches": [
120 0,
121 1,
122 2
123 ],
124 "stage2_branches": [
125 0,
126 1,
127 2
128 ],
129 "stage3_branches": [
130 0,
131 1,
132 2
133 ],
134 "fsd_dim": 72,
135 "fsd_blocks": 5,
136 "fsd_film_rank": 12,
137 "fsd_head_rank": 48,
138 "stage_projection_bottlenecks": [],
139 "teacher_init_checkpoint": null,
140 "teacher_init_method": "first",
141 "teacher_init": null,
142 "init_decoder_checkpoint": "artifacts/sub10m-search/root-a-piper-vits/en_US-amy-medium-retrain-20260712/a91b-decoder-10k/decoder-student.pt",
143 "decoder_init": {
144 "checkpoint": "artifacts/sub10m-search/root-a-piper-vits/en_US-amy-medium-retrain-20260712/a91b-decoder-10k/decoder-student.pt",
145 "decoder_parameters": 730721,
146 "matched_config": {
147 "in_channels": 192,
148 "channels": [
149 160,
150 80,
151 40,
152 20
153 ],
154 "res_layers": 1,
155 "variant": "piperlite",
156 "rank_ratio": 0.5,
157 "activation": "leaky_relu",
158 "stage_affine": false,
159 "factorized_pre_rank": 0,
160 "piper_res_factor_rank_ratio": 0.0,
161 "res_bank_scale_mode": "kept",
162 "stage0_branches": [
163 0,
164 1,
165 2
166 ],
167 "stage1_branches": [
168 0,
169 1,
170 2
171 ],
172 "stage2_branches": [
173 0,
174 1,
175 2
176 ],
177 "stage3_branches": [
178 0,
179 1,
180 2
181 ],
182 "fsd_dim": 72,
183 "fsd_blocks": 5,
184 "fsd_film_rank": 12,
185 "fsd_head_rank": 48,
186 "stage_projection_bottlenecks": [],
187 "post_filter_channels": 0,
188 "post_filter_layers": 0,
189 "post_filter_kernel": 9,
190 "post_filter_scale": 0.25,
191 "pre_tanh_repair_channels": 0,
192 "pre_tanh_repair_layers": 0,
193 "pre_tanh_repair_kernel": 7,
194 "pre_tanh_repair_scale": 0.15,
195 "istft_n_fft": 512
196 },
197 "allowed_post_filter_mismatches": [],
198 "allowed_pre_tanh_repair_mismatches": [],
199 "allowed_activation_mismatches": [],
200 "missing_keys": [],
201 "unexpected_keys": []
202 },
203 "post_filter_channels": 0,
204 "post_filter_layers": 0,
205 "post_filter_kernel": 9,
206 "post_filter_scale": 0.25,
207 "pre_tanh_repair_channels": 0,
208 "pre_tanh_repair_layers": 0,
209 "pre_tanh_repair_kernel": 7,
210 "pre_tanh_repair_scale": 0.15,
211 "istft_n_fft": 512,
212 "ap_amplitude_weight": 0.0,
213 "ap_phase_weight": 0.0,
214 "ap_complex_weight": 0.0,
215 "spectral_head_init": "default",
216 "spectral_head_init_scale": 0.001,
217 "ap_amp_init_bias": -6.0,
218 "ap_phase_real_init_bias": 1.0,
219 "spectral_head_init_summary": null,
220 "feature_exact_weight": 0.0,
221 "feature_exact_keys": [
222 "pre",
223 "up0"
224 ],
225 "feature_exact_teacher_init": null,
226 "signature_pack_dir": "artifacts/sub10m-search/root-a-piper-vits/en_US-amy-medium-retrain-20260712/a26-signatures-train512",
227 "exact_feature_pack_dir": null,
228 "signature_hint_weight": 0.05,
229 "signature_temporal_weight": 0.4,
230 "signature_phase_weight": 0.0,
231 "signature_phase_bins": 0,
232 "signature_keys": [
233 "stage0_mix",
234 "stage1_mix",
235 "stage2_mix",
236 "pre_tanh",
237 "audio"
238 ],
239 "bottleneck_code_checkpoint": null,
240 "bottleneck_code_weight": 0.0,
241 "bottleneck_code_keys": [],
242 "bottleneck_code_training_only_parameters": 0,
243 "echo_tail_weight": 0.0,
244 "echo_tail_min_ms": 8.0,
245 "echo_tail_max_ms": 40.0,
246 "echo_tail_lags": 9,
247 "echo_tail_margin": 0.02,
248 "paired_acoustic_residual_weight": 0.0,
249 "paired_acoustic_residual_max_scale": 0.25,
250 "lrc_pred_code_mix_prob": 0.0,
251 "lrc_pred_code_checkpoint": null,
252 "lrc_pred_code_residual_prob": 0.0,
253 "lrc_pred_code_residual_max_scale": 0.25,
254 "lrc_pred_code_cache": null
255 }
256 ```
257
258 Inputs:
259
260 - `latent`: float tensor, shape `[192, frames]`
261
262 Ops:
263
264 - PiperLite decoder blocks defined by the checkpoint config
265 - Conv1d same-padding pre/post projections
266 - compact upsampling stages with LeakyReLU for the current A91 branch
267 - low-rank/factorized residual projections when `rank_ratio` is present
268 - tanh waveform output
269
270 The decoder hop is `256` samples per latent frame and the output sample rate is
271 `22050` Hz.
272