{"repo": "avengermojo/mojoassistant", "files": {"config/known_runtimes.json": {"1d120796c29e9298707c7c0dae141fd9d5274d8f": {"t": 1785674485.4426634, "n": 12123, "h": "a5ef0db0b10bf9f6", "c": "{\n  \"_comment\": \"Known local LLM runtime catalog. Maintained by Rebecca's landscape scan. Do not edit manually — use the update process.\",\n  \"_last_updated\": \"2026-05-18\",\n  \"_update_task\": \"Queue Rebecca with goal: research local LLM runtimes landscape as of today\",\n  \"runtimes\": [\n    {\n      \"id\": \"lmstudio\",\n      \"label\": \"LM Studio\",\n      \"category\": \"gui\",\n      \"description\": \"Desktop GUI for running local LLMs. Easy model download, built-in chat, OpenAI-compatible server.\",\n      \"default_port\": 1234,\n      \"models_endpoint\": \"/v1/models\",\n      \"models_key\": \"data\",\n      \"install\": \"https://lmstudio.ai\",\n      \"gpu_support\": [\"nvidia\", \"amd\", \"apple_silicon\", \"cpu\"],\n      \"amd_rocm_support\": \"partial — depends on llama.cpp backend, some GPU detection issues on Linux\",\n      \"supported_modalities\": [\"text\", \"vision\", \"embedding\"],\n      \"model_formats\": [\"GGUF\", \"MLX\"],\n      \"popular_models\": [\"Qwen2.5-7B-Instruct\", \"Llama-3.1-8B-Instruct\", \"Mistral-7B\"],\n      \"new_features\": [\"improved model library browser\", \"multi-model serving\"],\n      \"warnings\": [\"Malware scare reported March 2026 — download only from official lmstudio.ai\"],\n      \"notes\": \"Best for beginners. GUI-first, server mode optional. Security incident March 2026 (v0.4.7+4 malware report — verify downloads).\"\n    },\n    {\n      \"id\": \"ollama\",\n      \"label\": \"Ollama\",\n      \"category\": \"cli_server\",\n      \"description\": \"One-command model runner. Pull and serve models via CLI. OpenAI-compatible API.\",\n      \"default_port\": 11434,\n      \"models_endpoint\": \"/api/tags\",\n      \"models_key\": \"models\",\n      \"install\": \"curl -fsSL https://ollama.ai/install.sh | sh\",\n      \"gpu_support\": [\"nvidia\", \"amd\", \"apple_silicon\", \"cpu\"],\n      \"amd_rocm_support\": \"good — via ROCm Docker images; some unsupported older GPUs need workarounds\",\n      \"supported_modalities\": [\"text\", \"vision\", \"embedding\"],\n      \"model_formats\": [\"GGUF\"],\n      \"popular_models\": [\"qwen2.5:7b\", \"llama3.1:8b\", \"phi3:mini\", \"gemma3:4b\"],\n      \"new_features\": [\"improved ROCm support via Docker\", \"vision model support expanded\"],\n      \"notes\": \"Best for quick setup. Huge model library. AMD users: use official ROCm Docker image.\"\n    },\n    {\n      \"id\": \"vllm\",\n      \"label\": \"vLLM\",\n      \"category\": \"inference_server\",\n      \"description\": \"High-throughput GPU inference server. Supports PagedAttention, continuous batching. Production-grade.\",\n      \"default_port\": 8000,\n      \"models_endpoint\": \"/v1/models\",\n      \"models_key\": \"data\",\n      \"install\": \"pip install vllm\",\n      \"gpu_support\": [\"nvidia\", \"amd\"],\n      \"amd_rocm_support\": \"good — ROCm install: pip install vllm --extra-index-url https://download.pytorch.org/whl/rocm6.2\",\n      \"amd_install\": \"pip install vllm --extra-index-url https://download.pytorch.org/whl/rocm6.2\",\n      \"supported_modalities\": [\"text\", \"vision\", \"audio\", \"embedding\"],\n      \"model_formats\": [\"safetensors\", \"AWQ\", \"GPTQ\", \"bfloat16\"],\n      \"popular_models\": [\"Qwen2.5-72B-Instruct-AWQ\", \"Llama-3.1-70B-AWQ\", \"Mistral-7B-Instruct\"],\n      \"new_features\": [\n        \"vLLM V1 core architecture rewrite (chunked prefill, zero-overhead scheduling)\",\n        \"FlashAttention-3 support\",\n        \"speculative decoding\",\n        \"LoRA serving\",\n        \"multi-modal (text + vision + audio)\"\n      ],\n      \"notes\": \"Best for large GPU servers. Can serve non-text models (speech, vision, audio) — always check modality of loaded model. Auth-protected endpoints common in production.\"\n    },\n    {\n      \"id\": \"sglang\",\n      \"label\": \"SGLang\",\n      \"category\": \"inference_server\",\n      \"description\": \"Structured generation language + fast runtime. RadixAttention for KV cache reuse. Strong for long-context.\",\n      \"default_port\": 30000,\n      \"models_endpoint\": \"/v1/models\",\n      \"models_key\": \"data\",\n      \"install\": \"pip install sglang[all]\",\n      \"gpu_support\": [\"nvidia\", \"amd\"],\n      \"amd_rocm_support\": \"growing — community reports, check GitHub for latest ROCm status\",\n      \"supported_modalities\": [\"text\", \"vision\"],\n      \"model_formats\": [\"safetensors\", \"AWQ\", \"bfloat16\"],\n      \"popular_models\": [\"Qwen2.5-72B\", \"Llama-3.1-70B\", \"DeepSeek-V3\", \"MiniMax-M2.5\"],\n      \"new_features\": [\n        \"RadixAttention (KV cache prefix reuse)\",\n        \"speculative decoding\",\n        \"MLA (Multi-head Latent Attention) for DeepSeek models\",\n        \"zero-overhead CPU scheduler\",\n        \"prefill-decode disaggregation\",\n        \"MiniMax M2.5 support (March 2026)\"\n      ],\n      \"notes\": \"Rapidly growing. Often cited as faster than vLLM for multi-turn conversations. Strong for long-context.\"\n    },\n    {\n      \"id\": \"lmdeploy\",\n      \"label\": \"LMDeploy\",\n      \"category\": \"inference_server\",\n      \"description\": \"InternLM team's inference framework. TurboMind engine, fast on NVIDIA. Good quantization support.\",\n      \"default_port\": 23333,\n      \"models_endpoint\": \"/v1/models\",\n      \"models_key\": \"data\",\n      \"install\": \"pip install lmdeploy\",\n      \"gpu_support\": [\"nvidia\"],\n      \"amd_rocm_support\": \"limited — primarily NVIDIA CUDA focused\",\n      \"supported_modalities\": [\"text\", \"vision\"],\n      \"model_formats\": [\"safetensors\", \"AWQ\", \"GPTQ\"],\n      \"popular_models\": [\"InternLM2.5-7B\", \"Qwen3 (added Feb 2026)\", \"Llama-3.1-8B\"],\n      \"new_features\": [\"Qwen3 support (Feb 2026)\", \"TurboMind engine optimizations\"],\n      \"warnings\": [\"CVE-2026-33626 — critical vulnerability exploited in the wild; run latest patched version (v0.12.3+) and do not expose to public internet\"],\n      \"notes\": \"Strong quantization toolchain. TurboMind engine often faster than vLLM for single-GPU NVIDIA. NVIDIA-only for production use.\"\n    },\n    {\n      \"id\": \"tabbyapi\",\n      \"label\": \"TabbyAPI\",\n      \"category\": \"inference_server\",\n      \"description\": \"Lightweight OpenAI-compatible server focused on GPTQ/EXL2 quantized models.\",\n      \"default_port\": 5000,\n      \"models_endpoint\": \"/v1/models\",\n      \"models_key\": \"data\",\n      \"install\": \"https://github.com/theroyallab/tabbyAPI\",\n      \"gpu_support\": [\"nvidia\"],\n      \"amd_rocm_support\": \"no — NVIDIA only (exllamav2 dependency)\",\n      \"supported_modalities\": [\"text\"],\n      \"model_formats\": [\"EXL2\", \"GPTQ\"],\n      \"popular_models\": [\"any EXL2 format model\"],\n      \"notes\": \"Best for EXL2 format models. Lower VRAM usage than vLLM for same quality. NVIDIA-only.\"\n    },\n    {\n      \"id\": \"unsloth\",\n      \"label\": \"Unsloth Studio\",\n      \"category\": \"finetune_and_serve\",\n      \"description\": \"Fine-tuning studio with inference. Optimized kernels for training and serving. GUI included.\",\n      \"default_port\": null,\n      \"models_endpoint\": null,\n      \"install\": \"https://unsloth.ai\",\n      \"detect_path\": \"~/.local/share/unsloth/launch-studio.sh\",\n      \"gpu_support\": [\"nvidia\", \"amd\"],\n      \"amd_rocm_support\": \"partial — fine-tuning primary use case; inference secondary\",\n      \"supported_modalities\": [\"text\", \"vision\"],\n      \"model_formats\": [\"GGUF\", \"safetensors\"],\n      \"popular_models\": [\"Qwen3.6 (documented May 2026)\", \"any GGUF model\"],\n      \"new_features\": [\"Qwen3.6 support\", \"optimized inference kernels\"],\n      \"notes\": \"Primary use: fine-tuning. Can serve fine-tuned models. Not a general-purpose inference server.\"\n    },\n    {\n      \"id\": \"llamacpp\",\n      \"label\": \"llama.cpp server\",\n      \"category\": \"cpu_and_gpu\",\n      \"description\": \"C++ inference engine. Runs on CPU, GPU, Apple Silicon. GGUF format. Widest hardware support.\",\n      \"default_port\": 8080,\n      \"models_endpoint\": \"/v1/models\",\n      \"models_key\": \"data\",\n      \"install\": \"https://github.com/ggml-org/llama.cpp\",\n      \"gpu_support\": [\"nvidia\", \"amd\", \"apple_silicon\", \"cpu\", \"vulkan\", \"npu\"],\n      \"amd_rocm_support\": \"good — via Vulkan backend; also ROCm/HIP; Strix Halo MTP benchmarks published\",\n      \"supported_modalities\": [\"text\", \"vision\", \"embedding\"],\n      \"model_formats\": [\"GGUF\"],\n      \"popular_models\": [\"any GGUF format model\"],\n      \"new_features\": [\n        \"MTP (Multi-Token Prediction) support — significant speed gain for supported models (27B much faster on Strix Halo)\",\n        \"tensor offloading optimization: offload tensors not layers for 200%+ speed improvement\",\n        \"Vulkan backend improvements\",\n        \"FlashAttention integration\"\n      ],\n      \"notes\": \"Widest hardware support. Q4_K_M recommended for quality/speed balance. Repo moved to ggml-org. MTP is significant speed gain for supported models.\"\n    },\n    {\n      \"id\": \"koboldcpp\",\n      \"label\": \"KoboldCpp\",\n      \"category\": \"gui_and_server\",\n      \"description\": \"llama.cpp wrapper with storytelling UI. Good for creative tasks and offline use.\",\n      \"default_port\": 5001,\n      \"models_endpoint\": \"/api/v1/model\",\n      \"install\": \"https://github.com/LostRuins/koboldcpp\",\n      \"gpu_support\": [\"nvidia\", \"amd\", \"cpu\", \"vulkan\"],\n      \"amd_rocm_support\": \"partial — via Vulkan backend\",\n      \"supported_modalities\": [\"text\"],\n      \"model_formats\": [\"GGUF\"],\n      \"notes\": \"Niche: creative writing and roleplay focus. Not recommended for agent tasks.\"\n    },\n    {\n      \"id\": \"lemonade\",\n      \"label\": \"Lemonade Server\",\n      \"category\": \"npu_and_gpu\",\n      \"description\": \"AMD-first omni-modal inference server. Auto-optimizes for AMD NPU (Ryzen AI) and GPU. Text, vision, audio.\",\n      \"default_port\": null,\n      \"models_endpoint\": null,\n      \"install\": \"https://lemonade-server.ai\",\n      \"install_alt\": \"https://github.com/lemonade-sdk/lemonade\",\n      \"gpu_support\": [\"amd\", \"nvidia\", \"cpu\", \"npu\"],\n      \"amd_rocm_support\": \"excellent — specifically targets AMD NPU (Ryzen AI) and GPU; auto-optimizes\",\n      \"supported_modalities\": [\"text\", \"vision\", \"audio\"],\n      \"model_formats\": [\"ONNX\", \"GGUF\"],\n      \"notes\": \"NEW in 2025/2026. Omni-modal (text, images, speech). Automatically optimizes for your GPU and NPU. Open source. Best choice for AMD Strix Halo / Ryzen AI Max users. Add to doctor detection.\"\n    }\n  ],\n  \"free_external_apis\": [\n    {\n      \"id\": \"openrouter\",\n      \"label\": \"OpenRouter\",\n      \"url\": \"https://openrouter.ai\",\n      \"free_tier\": true,\n      \"env_key\": \"OPEN_ROUTER_KEY\",\n      \"notes\": \"Largest model selection. Free tier available. Routes to many providers.\"\n    },\n    {\n      \"id\": \"groq\",\n      \"label\": \"Groq\",\n      \"url\": \"https://console.groq.com\",\n      \"free_tier\": true,\n      \"env_key\": \"GROQ_API_KEY\",\n      \"notes\": \"Very fast inference (LPU hardware). Free tier with rate limits. Good for quick tasks.\"\n    },\n    {\n      \"id\": \"cerebras\",\n      \"label\": \"Cerebras\",\n      \"url\": \"https://cloud.cerebras.ai\",\n      \"free_tier\": true,\n      \"env_key\": \"CEREBRAS_API_KEY\",\n      \"notes\": \"Generous free tier as of May 2026. Fast inference. Good fallback brain option.\"\n    },\n    {\n      \"id\": \"together\",\n      \"label\": \"Together AI\",\n      \"url\": \"https://api.together.xyz\",\n      \"free_tier\": true,\n      \"env_key\": \"TOGETHER_API_KEY\",\n      \"notes\": \"Wide model selection including open-weight models. Free credits on signup.\"\n    },\n    {\n      \"id\": \"github_models\",\n      \"label\": \"GitHub Models\",\n      \"url\": \"https://github.com/marketplace/models\",\n      \"free_tier\": true,\n      \"env_key\": \"GITHUB_TOKEN\",\n      \"notes\": \"Free via GitHub account. Limited but useful for bootstrapping.\"\n    }\n  ],\n  \"modality_notes\": \"Modern models are multimodal — a single endpoint may serve text+vision+thinking+tool-calling+audio+embedding. The 'supported_modalities' field per runtime describes what the runtime CAN serve, not what any specific loaded model supports. Auth-protected endpoints (e.g. vLLM in production) should be flagged as 'modality unknown' rather than assumed to be text-only.\",\n  \"needs_research\": [\n    \"Lemonade Server default port — not documented in May 2026 scan\",\n    \"EXL3 format future — TabbyAPI/EXL2 ecosystem watching community developments\",\n    \"FlashAttention-3 adoption status across runtimes beyond vLLM\",\n    \"Quantization format comparison: GGUF Q4_K_M vs EXL2 vs AWQ vs GPTQ quality/speed tradeoffs\",\n    \"MTP support matrix: which model families include MTP heads (Gemma 4 confirmed, others?)\"\n  ]\n}\n"}}}}