push

2026-08-22 03:36:34 -05:00
parent 1dffa3a0a2
commit 56a1152874
11 changed files with 109 additions and 30 deletions
@@ -1,26 +1,75 @@
 { ... }: {
  den.aspects.llama = {
-    nixos = { pkgs, ... }: {
-      services.llama-cpp = {
+    nixos = { pkgs, lib, ... }: {
+      environment.systemPackages = [ pkgs.cloudflared ];
+      environment.etc."cloudflared/9c1a18f9-a2c4-41bd-9a64-86ebb3e3283d.json".source = "/home/bug/.cloudflared/9c1a18f9-a2c4-41bd-9a64-86ebb3e3283d.json";
+
+      services.cloudflared = {
+        enable = true;
+        tunnels = {
+          "9c1a18f9-a2c4-41bd-9a64-86ebb3e3283d" = {
+            credentialsFile = "/etc/cloudflared/9c1a18f9-a2c4-41bd-9a64-86ebb3e3283d.json";
+
+            ingress = {
+              "llama.bug.tools" = "http://127.0.0.1:8080";
+            };
+
+            default = "http_status:404";
+          };
+        };
+      };
+
+      systemd.services."cloudflared-tunnel-9c1a18f9-a2c4-41bd-9a64-86ebb3e3283d".environment = {
+        TUNNEL_TRANSPORT_PROTOCOL = pkgs.lib.mkForce "http2";
+      };
+
+
+      services.llama-cpp = let
+        # https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md
+        modelsPreset = {
+          "*" = rec {
+            # keep-sorted start
+            agent = true;
+            cache-ram = 0; # unified memory
+            cache-type-k = "q8_0";
+            cache-type-v = "q8_0";
+            ctx-size = 256 * 256 * parallel;
+            fit = "off";
+            flash-attn = "on";
+            kv-unified = false;
+            mlock = true;
+            mmap = false;
+            n-gpu-layers = "all";
+            parallel = 1;
+            reasoning-preserve = true;
+            sleep-idle-seconds = -1;
+            spec-draft-type-k = "q8_0";
+            spec-draft-type-v = "q8_0";
+            # keep-sorted end
+            };
+
+          "qwen3.6-35b-a3b" = {
+            # keep-sorted start
+            hf-repo = "unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL";
+            min-p = 0.0;
+            temperature = 1.0;
+            top-k = 20;
+            top-p = 0.95;
+            # keep-sorted end
+          };
+        };
+      in {
        enable = true;
        package = pkgs.llama-cpp.override { cudaSupport = true; };
+
        settings = {
          host = "0.0.0.0";
          port = 8080;
          ctx-size = 65536;

-          models-preset = (pkgs.formats.ini { }).generate "llama-cpp-models-preset.ini" {
-            "Qwen3.6-35B-A3B" = {
-              hf-repo = "unsloth/Qwen3.6-35B-A3B-GGUF";
-              hf-file = "Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf";
-              alias = "qwen3.6-35b-a3b";
-              temp = "1.0";
-              top-p = "0.95";
-              top-k = "20";
-              n-gpu-layers = 80;
-              ctx-size = 65536;
-            };
-          };
+          models-max = 1;
+          models-preset = pkgs.writeText "llama-models.ini" (lib.generators.toINI { } modelsPreset);
+          no-models-autoload = true;
        };
      };
    };