29 lines
777 B
Nix
29 lines
777 B
Nix
{ ... }: {
|
|
den.aspects.llama = {
|
|
nixos = { pkgs, ... }: {
|
|
services.llama-cpp = {
|
|
enable = true;
|
|
package = pkgs.llama-cpp.override { cudaSupport = true; };
|
|
settings = {
|
|
host = "0.0.0.0";
|
|
port = 8080;
|
|
ctx-size = 65536;
|
|
|
|
models-preset = (pkgs.formats.ini { }).generate "llama-cpp-models-preset.ini" {
|
|
"Qwen3.6-35B-A3B" = {
|
|
hf-repo = "unsloth/Qwen3.6-35B-A3B-GGUF";
|
|
hf-file = "Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf";
|
|
alias = "qwen3.6-35b-a3b";
|
|
temp = "1.0";
|
|
top-p = "0.95";
|
|
top-k = "20";
|
|
n-gpu-layers = 80;
|
|
ctx-size = 65536;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|