push

2026-08-18 21:11:15 -05:00
parent c4220bc621
commit 1dffa3a0a2
23 changed files with 583 additions and 540 deletions
@@ -1,10 +1,27 @@
 { ... }: {
-  den.aspects.ollama = {
-    nixos = {
-      services.ollama = {
+  den.aspects.llama = {
+    nixos = { pkgs, ... }: {
+      services.llama-cpp = {
        enable = true;
+        package = pkgs.llama-cpp.override { cudaSupport = true; };
+        settings = {
+          host = "0.0.0.0";
+          port = 8080;
+          ctx-size = 65536;

-        loadModels = [ "llama3.2:3b" "deepseek-r1:1.5b" "deepseek-r1:8b" "qwen2.5-coder:7b" "qwen3.5:9b" "aratan/qwen3.5-agent-multimodal:9b" "qwen3-coder:30b" ];
+          models-preset = (pkgs.formats.ini { }).generate "llama-cpp-models-preset.ini" {
+            "Qwen3.6-35B-A3B" = {
+              hf-repo = "unsloth/Qwen3.6-35B-A3B-GGUF";
+              hf-file = "Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf";
+              alias = "qwen3.6-35b-a3b";
+              temp = "1.0";
+              top-p = "0.95";
+              top-k = "20";
+              n-gpu-layers = 80;
+              ctx-size = 65536;
+            };
+          };
+        };
      };
    };
  };