gguf.txt
933 B · 36 lines · plaintext Raw
1 # 1. Конвертация в bf16 (как договорились)
2 python convert_hf_to_gguf.py /mnt/nfs_share/Qwen3_8 \
3 --outfile /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.gguf --outtype bf16
4
5 # 2. Сборка llama-quantize (один раз)
6 cd /mnt/nfs_share/llama.cpp
7 cmake -B build
8 cmake --build build -j
9
10 # 3. Квантование в Q4_K_M
11 ./build/bin/llama-quantize \
12 /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.gguf \
13 /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.Q4_K_M.gguf \
14 Q4_K_M
15
16
17
18 cd /mnt/nfs_share/llama.cpp
19 cmake -B build -DGGML_CUDA=ON # для NVIDIA
20 make -C build
21
22
23 /mnt/nfs_share/llama.cpp/build/bin/llama-cli \
24 -m /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.Q4_K_M.gguf \
25 -n 512 \
26 -c 4096 \
27 -p "Explain quantum computing in simple terms:"
28
29
30
31
32
33 /mnt/nfs_share/llama.cpp/build/bin/llama-server \
34 -m /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.Q4_K_M.gguf \
35 --port 8000
36