{ ... }: { den.aspects.llama = { nixos = { pkgs, lib, ... }: { environment.systemPackages = [ pkgs.cloudflared ]; environment.etc."cloudflared/9c1a18f9-a2c4-41bd-9a64-86ebb3e3283d.json".source = "/home/bug/.cloudflared/9c1a18f9-a2c4-41bd-9a64-86ebb3e3283d.json"; services.cloudflared = { enable = true; tunnels = { "9c1a18f9-a2c4-41bd-9a64-86ebb3e3283d" = { credentialsFile = "/etc/cloudflared/9c1a18f9-a2c4-41bd-9a64-86ebb3e3283d.json"; ingress = { "llama.bug.tools" = "http://127.0.0.1:8080"; }; default = "http_status:404"; }; }; }; systemd.services."cloudflared-tunnel-9c1a18f9-a2c4-41bd-9a64-86ebb3e3283d".environment = { TUNNEL_TRANSPORT_PROTOCOL = pkgs.lib.mkForce "http2"; }; services.llama-cpp = let # https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md modelsPreset = { "*" = rec { # keep-sorted start agent = true; cache-ram = 0; # unified memory cache-type-k = "q8_0"; cache-type-v = "q8_0"; ctx-size = 256 * 256 * parallel; fit = "off"; flash-attn = "on"; kv-unified = false; mlock = true; mmap = false; n-gpu-layers = "all"; parallel = 1; reasoning-preserve = true; sleep-idle-seconds = -1; spec-draft-type-k = "q8_0"; spec-draft-type-v = "q8_0"; # keep-sorted end }; "qwen3.6-35b-a3b" = { # keep-sorted start hf-repo = "unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL"; min-p = 0.0; temperature = 1.0; top-k = 20; top-p = 0.95; # keep-sorted end }; }; in { enable = true; package = pkgs.llama-cpp.override { cudaSupport = true; }; settings = { host = "0.0.0.0"; port = 8080; ctx-size = 65536; models-max = 1; models-preset = pkgs.writeText "llama-models.ini" (lib.generators.toINI { } modelsPreset); no-models-autoload = true; }; }; }; }; }