vl/serve_decide.py
23.3 KB · 445 lines · python Raw
1 #!/usr/bin/env python3
2 """vLLM OpenAI-compatible server with a built-in System 1 endpoint (POST /v1/decide) and adaptive thinking.
3
4 One engine serves both systems, for JEV-27B / JEV-27B-VL (Qwen3.8) and GEV-26B-A4B (Gemma-4):
5 System 2 the unmodified base model, through the usual OpenAI endpoints (served model name)
6 System 1 the LoRA module "jev-decision" (backbone LoRA + decision head as an lm_head LoRA), through /v1/decide
7
8 POST /v1/decide
9 kind "noul" (yes/no), "score" (0-5) or "choice" (2-256 options)
10 state string, JSON object, or a list mixing text and images: ["text", {"image": "https://... | data:..."}]
11 question string
12 options list of strings (choice only)
13 strategy choices with more options than the head's trained choice slots (16 for JEV-27B / GEV; 256 for wide heads,
14 which always read up to 256 options in one pass): "single" (one pass, labels A-P then Q-Z, AA, ...),
15 "tournament" (groups of <=16 + a final of 16), "permute" (single pass over 4 option orders, averaged);
16 default per model
17 thinking "off" (default: System 1 only), "auto" (think only when the leading option is below `threshold`),
18 "on" (always think)
19 threshold System 1 confidence below which "auto" switches thinking on (default per model)
20 reasoning controls, as for the base model's own chat API:
21 chat_template_kwargs passed to the base model's chat template (e.g. Qwen3.8: {"reasoning_effort": "low"})
22 reasoning_effort shorthand for chat_template_kwargs.reasoning_effort, if the base model has it
23 (Qwen3.8 / JEV-27B: xhigh (default), medium, low; Gemma-4 / GEV: not available)
24 think_budget maximum thinking tokens; default: no limit beyond the context window
25 return_reasoning include System 2's reasoning text
26 debug include the System 1 and System 2 distributions
27
28 Adaptive thinking: System 1 gives p1 in one pass. If thinking is switched on, the base model reasons in its thinking
29 mode over the same state, question and options; when the thinking channel closes, the answer-letter distribution p2 is
30 read in one step, and the result is p = (1 - w) * p1 + w * p2 (default w = 0.5: the fast decision and the reasoning get
31 equal weight, which keeps the calibration of System 1; p2 alone is close to one-hot and over-confident).
32
33 Response (same shape as the hosted JEV API, plus "thinking" when requested):
34 {"kind", "effective_kind", "options", "probabilities", "choice_index", "choice", "adaptation", "protocol", "model",
35 "usage", "elapsed_seconds", "num_model_requests", "thinking": {...}}
36
37 Run it like `python -m vllm.entrypoints.openai.api_server` (same flags), with --lora-modules jev-decision=<adapter dir>
38 and --trust-request-chat-template. decision_head.json is read from the adapter directory, calibration.json from its
39 parent (override with JEV_DECIDE_CALIBRATION).
40 """
41 import asyncio
42 import json
43 import math
44 import os
45 import string
46 import time
47 import zlib
48 from typing import Literal
49
50 from fastapi import APIRouter, Request
51 from fastapi.responses import JSONResponse
52 from pydantic import BaseModel
53
54 import vllm.entrypoints.launchers.api_server.entry as entry
55 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
56 from vllm.entrypoints.openai.completion.protocol import CompletionRequest
57
58 LORA = os.environ.get("JEV_DECIDE_LORA", "jev-decision")
59 # JEV_DECIDE_SYSTEM2=0: the served model has no usable thinking mode (e.g. a pretrained base model without instruction
60 # tuning); thinking "auto"/"on" and system2_only are refused with 400.
61 SYSTEM2 = os.environ.get("JEV_DECIDE_SYSTEM2", "1") != "0"
62 MAX_OPTIONS = 256
63 CHUNK = 128 # vLLM caps logprob_token_ids at 128 per request
64 PROTOCOL = "jev27-bare-v1"
65 PLACEHOLDER = "XQXCONTENTXQX"
66
67 # Per-model settings. threshold / mix: adaptive-thinking defaults. GEV: fitted on 1,754 questions outside the Decision
68 # Index (CommonsenseQA, OpenBookQA, AQuA-RAT, MedMCQA, LogiQA, StrategyQA). JEV-27B: same defaults, not yet validated.
69 PROFILES = {
70 "qwen": {"prefix": "", "image": "<|vision_start|><|image_pad|><|vision_end|>", "end_think": "</think>",
71 "after_think": "\n\n", "strategy": "single", "threshold": 0.8, "mix": 0.5, "thinking": "off"},
72 "gemma": {"prefix": "<bos>", "image": "<|image|>", "end_think": "<channel|>", "after_think": "",
73 "strategy": "tournament", "threshold": 0.8, "mix": 0.5, "thinking": "off"},
74 }
75 # Read the full distribution: override generation_config defaults (top_k/top_p) that would truncate processed logprobs.
76 READ = dict(max_tokens=1, temperature=1.0, top_p=1.0, top_k=0, min_p=0.0, repetition_penalty=1.0,
77 add_special_tokens=False, return_tokens_as_token_ids=True)
78 S = {}
79
80
81 class DecideRequest(BaseModel):
82 kind: Literal["noul", "score", "choice"]
83 state: str | dict | list = ""
84 question: str
85 options: list[str] | None = None
86 strategy: Literal["auto", "single", "tournament", "permute"] = "auto"
87 thinking: Literal["default", "off", "auto", "on"] = "default"
88 threshold: float | None = None
89 think_budget: int | None = None
90 reasoning_effort: str | None = None
91 chat_template_kwargs: dict | None = None
92 return_reasoning: bool = False
93 debug: bool = False
94 system2_only: bool = False # skip System 1: think and return System 2's answer distribution (for callers that already have p1)
95
96
97 # ------------------------------------------------------------------ setup
98 def setup(args):
99 from transformers import AutoConfig, AutoTokenizer
100
101 path = next((m.path for m in (args.lora_modules or []) if m.name == LORA), None)
102 if path is None:
103 raise SystemExit(f"serve_decide: start vLLM with --lora-modules {LORA}=<adapter dir>")
104 head = json.load(open(os.path.join(path, "decision_head.json")))
105 calib = os.environ.get("JEV_DECIDE_CALIBRATION") or os.path.join(os.path.dirname(os.path.abspath(path)), "calibration.json")
106 temps = json.load(open(calib))["per_kind"] if os.path.exists(calib) else {"noul": 1.0, "score": 1.0, "choice": 1.0}
107 temps.setdefault("score", 1.0)
108 src = args.tokenizer or args.model
109 tok = AutoTokenizer.from_pretrained(src, trust_remote_code=args.trust_remote_code)
110 mtype = AutoConfig.from_pretrained(args.model, trust_remote_code=args.trust_remote_code).model_type
111 prof = dict(PROFILES["gemma" if "gemma" in mtype else "qwen"])
112 for k in ("strategy", "threshold", "mix", "thinking"):
113 if os.environ.get(f"JEV_DECIDE_{k.upper()}"):
114 v = os.environ[f"JEV_DECIDE_{k.upper()}"]
115 prof[k] = v if k in ("strategy", "thinking") else float(v)
116
117 def single_token_labels(context):
118 out = []
119 for lab in list(string.ascii_uppercase) + [a + b for a in string.ascii_uppercase for b in string.ascii_uppercase]:
120 t = tok.encode(lab, add_special_tokens=False)
121 if len(t) == 1 and t[0] in tok.encode(context.format(lab), add_special_tokens=False):
122 out.append((lab, t[0]))
123 if len(out) == MAX_OPTIONS:
124 break
125 return out
126
127 dec = single_token_labels("x\n{}) y") # System 1 option-line labels
128 lo, hi = head["slots"]["ranges"]["choice"]
129 assert [t for _, t in dec[: hi - lo]] == head["verbalizer_ids"][lo:hi], "first labels must be the trained A-P head"
130 base = tok.encode("Answer: (", add_special_tokens=False) # System 2 answer labels
131 ans = []
132 for lab in list(string.ascii_uppercase) + [a + b for a in string.ascii_uppercase for b in string.ascii_uppercase]:
133 ids = tok.encode(f"Answer: ({lab})", add_special_tokens=False)
134 if ids[: len(base)] == base and len(ids) == len(base) + 2:
135 ans.append((lab, ids[len(base)]))
136 if len(ans) == MAX_OPTIONS:
137 break
138 chat = tok.apply_chat_template([{"role": "user", "content": PLACEHOLDER}], tokenize=False, add_generation_prompt=True,
139 enable_thinking=True)
140 pre, post = chat.split(PLACEHOLDER)
141 raw = ("{%- for m in messages -%}{%- if m['content'] is string -%}{{ m['content'] }}{%- else -%}"
142 "{%- for c in m['content'] -%}{%- if c['type'] == 'text' -%}{{ c['text'] }}{%- else -%}" + prof["image"] +
143 "{%- endif -%}{%- endfor -%}{%- endif -%}{%- endfor -%}")
144 names = args.served_model_name
145 S.update(tok=tok, has_effort="reasoning_effort" in (tok.chat_template or ""), think_cache={},
146 max_logprobs=int(getattr(args, "max_logprobs", 20) or 20),
147 head=head, temps=temps, labels=[l for l, _ in dec], label_ids=[t for _, t in dec], ans_labels=ans, prof=prof,
148 think_pre=pre, think_post=post, raw=raw, end_think_id=tok.convert_tokens_to_ids(prof["end_think"]),
149 model=(names[0] if isinstance(names, list) else names) or args.model, model_type=mtype)
150
151
152 # ------------------------------------------------------------------ helpers
153 def _parts(state):
154 if isinstance(state, str):
155 return [{"type": "text", "text": state}], False
156 if isinstance(state, dict):
157 return [{"type": "text", "text": json.dumps(state, ensure_ascii=False)}], False
158 out, img = [], False
159 for p in state:
160 if isinstance(p, str):
161 out.append({"type": "text", "text": p})
162 elif isinstance(p, dict) and "image" in p:
163 out.append({"type": "image_url", "image_url": {"url": p["image"]}}); img = True
164 elif isinstance(p, dict) and p.get("type") == "image_url":
165 out.append(p); img = True
166 elif isinstance(p, dict) and p.get("type") == "text":
167 out.append(p)
168 else:
169 out.append({"type": "text", "text": json.dumps(p, ensure_ascii=False)})
170 return out, img
171
172
173 def _softmax(z):
174 m = max(z); e = [math.exp(x - m) for x in z]; s = sum(e)
175 return [x / s for x in e]
176
177
178 def _err(msg, code=400):
179 return JSONResponse({"error": {"message": msg, "type": "BadRequestError", "code": code}}, status_code=code)
180
181
182 class Upstream(Exception):
183 def __init__(self, resp):
184 self.resp = resp
185
186
187 def _check(out):
188 if hasattr(out, "error"):
189 raise Upstream(out)
190 return out
191
192
193 class Ctx:
194 def __init__(self, raw):
195 self.raw, self.requests, self.prompt_tokens, self.completion_tokens = raw, 0, 0, 0
196
197 def count(self, out):
198 self.requests += 1
199 if getattr(out, "usage", None):
200 self.prompt_tokens += out.usage.prompt_tokens or 0
201 self.completion_tokens += out.usage.completion_tokens or 0
202
203
204 def _readout_allowed(ids):
205 """Read-out path: restrict the next token to `ids` and return all of them as top logprobs when the server allows that
206 many (--max-logprobs); otherwise ask for `ids` explicitly with logprob_token_ids (chunks of 128). The first path also
207 works with speculative decoding; logprob_token_ids does not in this vLLM build. Both give the same distribution."""
208 return ids if len(ids) <= S["max_logprobs"] else None
209
210
211 async def _logprobs(ctx, content, has_img, ids, model, allowed=None):
212 """One-token read-out with a fallback to the other read-out path if the first one fails."""
213 try:
214 return await _logprobs_once(ctx, content, has_img, ids, model, allowed)
215 except Upstream:
216 raise
217 except Exception:
218 other = None if allowed is not None else (ids if len(ids) <= S["max_logprobs"] else None)
219 if other is None and allowed is None:
220 raise
221 return await _logprobs_once(ctx, content, has_img, ids, model, other)
222
223
224 async def _logprobs_once(ctx, content, has_img, ids, model, allowed=None):
225 """One-token read-out: {token_id: logprob} for `ids` after the content (text or chat parts)."""
226 lp = {}
227 step = CHUNK if allowed is None else len(ids)
228 for i in range(0, len(ids), step):
229 chunk = ids[i: i + step]
230 kw = dict(logprob_token_ids=chunk) if allowed is None else dict(allowed_token_ids=allowed)
231 if has_img:
232 r = ChatCompletionRequest(model=model, messages=[{"role": "user", "content": content}], chat_template=S["raw"],
233 add_generation_prompt=False, logprobs=True, top_logprobs=len(chunk) if allowed else 1, **kw, **READ)
234 out = _check(await ctx.raw.app.state.openai_serving_chat.create_chat_completion(r, ctx.raw)); ctx.count(out)
235 lp.update({int(t.token.split(":")[1]): t.logprob for t in out.choices[0].logprobs.content[0].top_logprobs})
236 else:
237 text = "".join(c["text"] for c in content)
238 r = CompletionRequest(model=model, prompt=text, logprobs=len(chunk) if allowed else 1, **kw, **READ)
239 out = _check(await ctx.raw.app.state.openai_serving_completion.create_completion(r, ctx.raw)); ctx.count(out)
240 lp.update({int(k.split(":")[1]): v for k, v in out.choices[0].logprobs.top_logprobs[0].items()})
241 if allowed is not None:
242 break
243 return lp
244
245
246 # ------------------------------------------------------------------ System 1
247 async def s1_pass(ctx, kind, parts, has_img, question, opts):
248 """One System 1 pass over `opts` (<=256). Returns probabilities aligned with opts."""
249 head, temps, prof = S["head"], S["temps"], S["prof"]
250 lo, hi = head["slots"]["ranges"][kind]
251 if kind == "choice":
252 n = len(opts); ids = S["label_ids"][:n]
253 bias = [head["bias"][lo + i] if lo + i < hi else 0.0 for i in range(n)]
254 lines = [f"{S['labels'][i]}) {o}" for i, o in enumerate(opts)]
255 else:
256 ids = head["verbalizer_ids"][lo:hi]; bias = head["bias"][lo:hi]; lines = opts
257 content = ([{"type": "text", "text": f"{prof['prefix']}[kind] {kind}\n[state] "}] + parts +
258 [{"type": "text", "text": f"\n[question] {question}\n[options]\n" + "\n".join(lines) + "\n[decision]:"}])
259 lp = await _logprobs(ctx, content, has_img, ids, LORA, allowed=_readout_allowed(ids))
260 return _softmax([(max(lp.get(t, -1e9), -1e9) + b) / temps[kind] for t, b in zip(ids, bias)])
261
262
263 def _groups(n, k=16):
264 g = math.ceil(n / k); base, extra = divmod(n, g); out, i = [], 0
265 for j in range(g):
266 size = base + (1 if j < extra else 0); out.append(list(range(i, i + size))); i += size
267 return out
268
269
270 def _native():
271 lo, hi = S["head"]["slots"]["ranges"]["choice"]
272 return hi - lo # trained choice slots: 16 (JEV-27B, GEV) or 256 (wide heads)
273
274
275 async def s1_dist(ctx, kind, parts, has_img, question, opts, strategy):
276 if kind != "choice" or len(opts) <= _native() or strategy == "single":
277 return await s1_pass(ctx, kind, parts, has_img, question, opts)
278 n = len(opts)
279 if strategy == "permute":
280 import random
281 rng = random.Random(zlib.crc32(question.encode()))
282 orders = [list(range(n))] + [rng.sample(range(n), n) for _ in range(3)]
283 res = await asyncio.gather(*(s1_pass(ctx, kind, parts, has_img, question, [opts[i] for i in o]) for o in orders))
284 p = [0.0] * n
285 for o, r in zip(orders, res):
286 for i, v in zip(o, r):
287 p[i] += v / len(orders)
288 return p
289 groups = _groups(n) # tournament: groups of <=16 in the given order (in parallel), then a final of 16
290 parts_g = await asyncio.gather(*(s1_pass(ctx, kind, parts, has_img, question, [opts[i] for i in g]) for g in groups))
291 in_group = {o: p for g, ps in zip(groups, parts_g) for o, p in zip(g, ps)}
292 chosen = [max(g, key=lambda o: (in_group[o], -o)) for g in groups]
293 rest = sorted((o for g in groups for o in g if o not in set(chosen)), key=lambda o: (-in_group[o], o))
294 fin = sorted(chosen + rest[: max(0, 16 - len(chosen))])
295 final = dict(zip(fin, await s1_pass(ctx, kind, parts, has_img, question, [opts[i] for i in fin])))
296 group_of = {o: gi for gi, g in enumerate(groups) for o in g}
297 share = [0.0] * len(groups); cap = [0.0] * len(groups)
298 for f in fin:
299 share[group_of[f]] += final[f]; cap[group_of[f]] += in_group[f]
300 among = sum(a * b for a, b in zip(share, cap))
301 p = [final[o] * among if o in final else share[group_of[o]] * in_group[o] for o in range(n)]
302 s = sum(p)
303 return [x / s for x in p]
304
305
306 # ------------------------------------------------------------------ System 2 (thinking)
307 def _think_frame(kwargs):
308 """Chat-template prefix / suffix around the user turn, thinking on, with the caller's template kwargs (cached)."""
309 key = json.dumps(kwargs or {}, sort_keys=True)
310 if key not in S["think_cache"]:
311 kw = {k: v for k, v in (kwargs or {}).items() if k != "enable_thinking"}
312 chat = S["tok"].apply_chat_template([{"role": "user", "content": PLACEHOLDER}], tokenize=False, add_generation_prompt=True,
313 enable_thinking=True, **kw)
314 S["think_cache"][key] = tuple(chat.split(PLACEHOLDER))
315 return S["think_cache"][key]
316
317
318 async def s2_dist(ctx, kind, parts, has_img, question, opts, budget, want_text, tmpl_kwargs=None):
319 """The base model thinks over the same input; the answer-letter distribution is read after the thinking channel."""
320 shown = ["Yes (true)", "No (false)"] if kind == "noul" else opts
321 labs = S["ans_labels"][: len(shown)]
322 body = "\n".join(f"({l}) {o}" for (l, _), o in zip(labs, shown))
323 tail = (f"\n\nQuestion: {question}\n\nOptions:\n{body}\n\n"
324 "Think it through carefully, then give your final answer on the last line in the form: Answer: (X)")
325 pre, post = _think_frame(tmpl_kwargs)
326 user = [{"type": "text", "text": pre}] + parts + [{"type": "text", "text": tail + post}]
327 seed = zlib.crc32((question + body).encode()) & 0x7FFFFFFF
328 t0 = time.time()
329 gen = ChatCompletionRequest(model=S["model"], messages=[{"role": "user", "content": user}], chat_template=S["raw"],
330 add_generation_prompt=False, add_special_tokens=False, max_tokens=budget, seed=seed, # None: up to the context window
331 stop_token_ids=[S["end_think_id"]], skip_special_tokens=False)
332 out = _check(await ctx.raw.app.state.openai_serving_chat.create_chat_completion(gen, ctx.raw)); ctx.count(out)
333 thought = out.choices[0].message.content or ""
334 finished = out.choices[0].finish_reason == "stop"
335 ntok = out.usage.completion_tokens if out.usage else None
336 read = user + [{"type": "text", "text": thought + S["prof"]["end_think"] + S["prof"]["after_think"] + "Answer: ("}]
337 ids = [t for _, t in labs]
338 lp = await _logprobs(ctx, read, True, ids, S["model"], allowed=_readout_allowed(ids))
339 p = _softmax([max(lp.get(t, -1e9), -1e9) for t in ids])
340 if kind == "noul":
341 p = [p[1], p[0]] # (A) yes / (B) no -> [P(false), P(true)]
342 info = {"think_tokens": ntok, "think_seconds": round(time.time() - t0, 3), "finished_within_budget": finished}
343 if want_text:
344 info["reasoning"] = thought.replace("<|channel>thought\n", "").strip()
345 return p, info
346
347
348 def _fold(p1, p2, w):
349 return [(1 - w) * x + w * y for x, y in zip(p1, p2)]
350
351
352 # ------------------------------------------------------------------ routes
353 router = APIRouter()
354
355
356 @router.get("/v1/decide/info")
357 async def info():
358 p = S["prof"]
359 return {"protocol": PROTOCOL, "model": S["model"], "model_type": S["model_type"], "max_options": len(S["labels"]),
360 "native_choice_options": len(S["head"]["slots"]["verbalizers"]) - 8, "temperatures": S["temps"],
361 "defaults": {"strategy": p["strategy"], "thinking": p["thinking"], "threshold": p["threshold"], "mix": p["mix"],
362 "think_budget": None},
363 "system2_available": SYSTEM2,
364 "reasoning_controls": {"enable_thinking": True, "reasoning_effort": ["xhigh", "medium", "low"] if S["has_effort"] else None,
365 "think_budget": "max thinking tokens (default: up to the context window)"}}
366
367
368 @router.post("/v1/decide")
369 async def decide(req: DecideRequest, raw: Request):
370 t0 = time.time()
371 if req.kind == "choice":
372 opts = req.options or []
373 if not 2 <= len(opts) <= len(S["labels"]):
374 return _err(f"choice needs 2-{len(S['labels'])} options, got {len(opts)}")
375 else:
376 opts = ["false", "true"] if req.kind == "noul" else [str(i) for i in range(6)]
377 prof = S["prof"]
378 thinking = prof["thinking"] if req.thinking == "default" else req.thinking
379 if req.kind == "score":
380 if req.thinking in ("auto", "on"):
381 return _err("thinking is supported for noul and choice")
382 thinking = "off"
383 tmpl = dict(req.chat_template_kwargs or {})
384 if req.reasoning_effort is not None:
385 if not S["has_effort"]:
386 return _err("this base model has no reasoning_effort setting; control the reasoning length with think_budget")
387 tmpl["reasoning_effort"] = req.reasoning_effort
388 strategy = prof["strategy"] if req.strategy == "auto" else req.strategy
389 tau = prof["threshold"] if req.threshold is None else req.threshold
390 parts, has_img = _parts(req.state)
391 ctx = Ctx(raw)
392 if not SYSTEM2 and (thinking in ("auto", "on") or req.system2_only):
393 return _err("thinking (System 2) is not available for this model: it is a pretrained base model without "
394 "instruction tuning; use thinking \"off\"")
395 try:
396 if req.system2_only:
397 if req.kind == "score":
398 return _err("system2_only is supported for noul and choice")
399 p2, think = await s2_dist(ctx, req.kind, parts, has_img, req.question, opts, req.think_budget, req.return_reasoning, tmpl)
400 k = max(range(len(p2)), key=p2.__getitem__)
401 return {"kind": req.kind, "options": opts, "probabilities": p2, "choice_index": k, "choice": opts[k], "system": 2,
402 "model": S["model"], "thinking": {"used": True, "budget": req.think_budget, **think},
403 "usage": {"prompt_tokens": ctx.prompt_tokens, "completion_tokens": ctx.completion_tokens},
404 "elapsed_seconds": time.time() - t0, "num_model_requests": ctx.requests}
405 p1 = await s1_dist(ctx, req.kind, parts, has_img, req.question, opts, strategy)
406 probs, think = p1, None
407 if thinking == "on" or (thinking == "auto" and max(p1) < tau):
408 p2, think = await s2_dist(ctx, req.kind, parts, has_img, req.question, opts, req.think_budget, req.return_reasoning, tmpl)
409 probs = _fold(p1, p2, prof["mix"])
410 think = {"used": True, **think}
411 if req.debug:
412 think.update(system1=p1, system2=p2)
413 elif thinking != "off":
414 think = {"used": False}
415 if req.debug:
416 think["system1"] = p1
417 except Upstream as e:
418 return JSONResponse(e.resp.model_dump(), status_code=e.resp.error.code)
419 k = max(range(len(probs)), key=probs.__getitem__)
420 native = req.kind != "choice" or len(opts) <= _native()
421 resp = {"kind": req.kind, "effective_kind": req.kind, "options": opts, "probabilities": probs, "choice_index": k,
422 "choice": opts[k], "adaptation": "native" if native else f"{strategy}", "protocol": PROTOCOL, "model": S["model"],
423 "usage": {"prompt_tokens": ctx.prompt_tokens, "completion_tokens": ctx.completion_tokens,
424 "total_tokens": ctx.prompt_tokens + ctx.completion_tokens},
425 "elapsed_seconds": time.time() - t0, "num_model_requests": ctx.requests}
426 if think is not None:
427 resp["thinking"] = {"mode": thinking, "threshold": tau, "budget": req.think_budget, **({"chat_template_kwargs": tmpl} if tmpl else {}), **think}
428 return resp
429
430
431 _build_app = entry.build_app
432
433
434 def build_app(args, *a, **kw):
435 app = _build_app(args, *a, **kw)
436 setup(args)
437 app.include_router(router)
438 return app
439
440
441 entry.build_app = build_app
442
443 if __name__ == "__main__":
444 entry.main()
445