serve.sh
1.0 KB · 14 lines · bash Raw
1 #!/usr/bin/env bash
2 # Serve autotrust/JEV-35B with vLLM: System 1 typed decisions (POST /v1/decide), text and images.
3 # The System 1 decision model is merged into the weights (no LoRA): the head bias and per-kind temperatures are applied
4 # by serve_decide.py (adapter_vllm/decision_head.json, calibration.json). serve_decide.py is the standard vLLM OpenAI
5 # server (same flags) with /v1/decide added. MAX_MODEL_LEN: up to 262144 (the backbone's native context).
6 set -e
7 MODEL_DIR=${MODEL_DIR:-JEV-35B}
8 [ -d "$MODEL_DIR" ] || hf download autotrust/JEV-35B --local-dir "$MODEL_DIR"
9 export JEV_DECIDE_MERGED_HEAD="$MODEL_DIR/adapter_vllm/decision_head.json" JEV_DECIDE_SYSTEM2=0
10 exec python3 "$MODEL_DIR/serve_decide.py" --model "$MODEL_DIR" --served-model-name autotrust/JEV-35B \
11 --logprobs-mode processed_logprobs --max-logprobs 256 --max-model-len ${MAX_MODEL_LEN:-65536} \
12 --enable-prefix-caching --mamba-cache-mode align --max-num-seqs ${MAX_NUM_SEQS:-64} --trust-request-chat-template \
13 --limit-mm-per-prompt '{"image": 8}' --port ${PORT:-8000}
14