78 lines
2.3 KiB
Nix
78 lines
2.3 KiB
Nix
{ ... }: {
|
|
den.aspects.llama = {
|
|
nixos = { pkgs, lib, ... }: {
|
|
environment.systemPackages = [ pkgs.cloudflared ];
|
|
environment.etc."cloudflared/608ba37a-1be8-49e8-be2d-abeb8b77081d.json".source = "/home/bug/.cloudflared/608ba37a-1be8-49e8-be2d-abeb8b77081d.json";
|
|
|
|
services.cloudflared = {
|
|
enable = true;
|
|
tunnels = {
|
|
"608ba37a-1be8-49e8-be2d-abeb8b77081d" = {
|
|
credentialsFile = "/etc/cloudflared/608ba37a-1be8-49e8-be2d-abeb8b77081d.json";
|
|
|
|
ingress = {
|
|
"llama.bug.tools" = "http://127.0.0.1:8080";
|
|
};
|
|
|
|
default = "http_status:404";
|
|
};
|
|
};
|
|
};
|
|
|
|
systemd.services."cloudflared-tunnel-608ba37a-1be8-49e8-be2d-abeb8b77081d".environment = {
|
|
TUNNEL_TRANSPORT_PROTOCOL = pkgs.lib.mkForce "http2";
|
|
};
|
|
|
|
|
|
services.llama-cpp = let
|
|
# https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md
|
|
modelsPreset = {
|
|
"*" = rec {
|
|
# keep-sorted start
|
|
agent = true;
|
|
cache-ram = 0; # unified memory
|
|
cache-type-k = "q8_0";
|
|
cache-type-v = "q8_0";
|
|
ctx-size = 256 * 256 * parallel;
|
|
fit = "off";
|
|
flash-attn = "on";
|
|
kv-unified = false;
|
|
mlock = true;
|
|
mmap = false;
|
|
n-gpu-layers = "all";
|
|
parallel = 1;
|
|
reasoning-preserve = true;
|
|
sleep-idle-seconds = -1;
|
|
spec-draft-type-k = "q8_0";
|
|
spec-draft-type-v = "q8_0";
|
|
# keep-sorted end
|
|
};
|
|
|
|
"qwen3.6-35b-a3b" = {
|
|
# keep-sorted start
|
|
hf-repo = "unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL";
|
|
min-p = 0.0;
|
|
temperature = 1.0;
|
|
top-k = 20;
|
|
top-p = 0.95;
|
|
# keep-sorted end
|
|
};
|
|
};
|
|
in {
|
|
enable = true;
|
|
package = pkgs.llama-cpp.override { cudaSupport = true; };
|
|
|
|
settings = {
|
|
host = "0.0.0.0";
|
|
port = 8080;
|
|
ctx-size = 65536;
|
|
|
|
models-max = 1;
|
|
models-preset = pkgs.writeText "llama-models.ini" (lib.generators.toINI { } modelsPreset);
|
|
no-models-autoload = true;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|