push

2026-08-22 21:46:30 -05:00
parent 022d4ab1af
commit cbbd94a130
8 changed files with 184 additions and 30 deletions
@@ -0,0 +1,77 @@
+{ ... }: {
+  den.aspects.llama = {
+    nixos = { pkgs, lib, ... }: {
+      environment.systemPackages = [ pkgs.cloudflared ];
+      environment.etc."cloudflared/9c1a18f9-a2c4-41bd-9a64-86ebb3e3283d.json".source = "/home/bug/.cloudflared/9c1a18f9-a2c4-41bd-9a64-86ebb3e3283d.json";
+
+      services.cloudflared = {
+        enable = true;
+        tunnels = {
+          "9c1a18f9-a2c4-41bd-9a64-86ebb3e3283d" = {
+            credentialsFile = "/etc/cloudflared/9c1a18f9-a2c4-41bd-9a64-86ebb3e3283d.json";
+
+            ingress = {
+              "llama.bug.tools" = "http://127.0.0.1:8080";
+            };
+
+            default = "http_status:404";
+          };
+        };
+      };
+
+      systemd.services."cloudflared-tunnel-9c1a18f9-a2c4-41bd-9a64-86ebb3e3283d".environment = {
+        TUNNEL_TRANSPORT_PROTOCOL = pkgs.lib.mkForce "http2";
+      };
+
+
+      services.llama-cpp = let
+        # https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md
+        modelsPreset = {
+          "*" = rec {
+            # keep-sorted start
+            agent = true;
+            cache-ram = 0; # unified memory
+            cache-type-k = "q8_0";
+            cache-type-v = "q8_0";
+            ctx-size = 256 * 256 * parallel;
+            fit = "off";
+            flash-attn = "on";
+            kv-unified = false;
+            mlock = true;
+            mmap = false;
+            n-gpu-layers = "all";
+            parallel = 1;
+            reasoning-preserve = true;
+            sleep-idle-seconds = -1;
+            spec-draft-type-k = "q8_0";
+            spec-draft-type-v = "q8_0";
+            # keep-sorted end
+            };
+
+          "qwen3.6-35b-a3b" = {
+            # keep-sorted start
+            hf-repo = "unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL";
+            min-p = 0.0;
+            temperature = 1.0;
+            top-k = 20;
+            top-p = 0.95;
+            # keep-sorted end
+          };
+        };
+      in {
+        enable = true;
+        package = pkgs.llama-cpp.override { cudaSupport = true; };
+
+        settings = {
+          host = "0.0.0.0";
+          port = 8080;
+          ctx-size = 65536;
+
+          models-max = 1;
+          models-preset = pkgs.writeText "llama-models.ini" (lib.generators.toINI { } modelsPreset);
+          no-models-autoload = true;
+        };
+      };
+    };
+  };
+}