Blaizzy/mlx-vlm

pip install -U mlx-vlm

pip install -U 'mlx-vlm[ui]'

python3 skills/scripts/validate_skills.py

# Claude Code
/plugin marketplace add /path/to/mlx-vlm
/plugin install mlx-vlm-skills@mlx-vlm

# Codex CLI
codex plugin marketplace add /path/to/mlx-vlm
codex plugin add mlx-vlm-skills@mlx-vlm

# Gemini CLI
gemini extensions install /path/to/mlx-vlm/skills

# Text generation
mlx_vlm.generate --model mlx-community/Qwen2-VL-2B-Instruct-4bit --max-tokens 100 --prompt "Hello, how are you?"

# Image generation
mlx_vlm.generate --model mlx-community/Qwen2-VL-2B-Instruct-4bit --max-tokens 100 --temperature 0.0 --image http://images.cocodataset.org/val2017/000000039769.jpg

# Audio generation (New)
mlx_vlm.generate --model mlx-community/gemma-3n-E2B-it-4bit --max-tokens 100 --prompt "Describe what you hear" --audio /path/to/audio.wav

# Multi-modal generation (Image + Audio)
mlx_vlm.generate --model mlx-community/gemma-3n-E2B-it-4bit --max-tokens 100 --prompt "Describe what you see and hear" --image /path/to/image.jpg --audio /path/to/audio.wav

mlx_vlm.generate --model mlx-community/Qwen3.5-2B-4bit \
  --thinking-budget 50 \
  --thinking-start-token "<think>" \
  --thinking-end-token "</think>" \
  --enable-thinking \
  --prompt "Solve 2+2"

mlx_vlm.server --model Qwen/Qwen3.5-4B --enable-thinking

mlx_vlm.server --model Qwen/Qwen3.5-4B \
  --enable-thinking \
  --thinking-budget 512 \
  --thinking-start-token "<think>" \
  --thinking-end-token "</think>"

# Text generation with speculative decoding
mlx_vlm.generate --model Qwen/Qwen3.5-4B \
  --draft-model z-lab/Qwen3.5-4B-DFlash \
  --prompt "Write a quicksort in Python." \
  --max-tokens 512 --temperature 0 --enable-thinking

# Also works with images
mlx_vlm.generate --model Qwen/Qwen3.5-4B \
  --draft-model z-lab/Qwen3.5-4B-DFlash \
  --image examples/images/cats.jpg \
  --prompt "Describe this image." \
  --max-tokens 256 --temperature 0 --enable-thinking

# Server with speculative decoding
mlx_vlm.server --model Qwen/Qwen3.5-4B \
  --draft-model z-lab/Qwen3.5-4B-DFlash

from mlx_vlm import load
from mlx_vlm.generate import generate
from mlx_vlm.speculative.drafters import load_drafter

model, processor = load("Qwen/Qwen3.5-4B")
draft_model, draft_kind = load_drafter("z-lab/Qwen3.5-4B-DFlash")
draft_model.config.draft_window_size = 256  # None disables windowing

result = generate(
    model,
    processor,
    "Write a quicksort in Python.",
    max_tokens=512,
    temperature=0,
    draft_model=draft_model,
    draft_kind=draft_kind,
)

mlx_vlm.generate --model mlx-community/gemma-4-31B-it-bf16 \
  --draft-model mlx-community/gemma-4-31B-it-assistant-bf16 \
  --draft-kind mtp --draft-block-size 4 \
  --prompt "Explain speculative decoding in 3 sentences." \
  --max-tokens 256 --temperature 0

# Server
mlx_vlm.server --model mlx-community/gemma-4-31B-it-bf16 \
  --draft-model mlx-community/gemma-4-31B-it-assistant-bf16 \
  --draft-kind mtp --draft-block-size 4

mlx_vlm.generate --model mlx-community/gemma-4-31B-it-bf16 \
  --draft-model RedHatAI/gemma-4-31B-it-speculator.eagle3 \
  --prompt "Explain speculative decoding in 3 sentences." \
  --max-tokens 256 --temperature 0

# Server
mlx_vlm.server --model mlx-community/gemma-4-31B-it-bf16 \
  --draft-model RedHatAI/gemma-4-31B-it-speculator.eagle3

mlx_vlm.convert \
  --hf-path MiniMaxAI/MiniMax-M3 \
  --mlx-path ~/MiniMax-M3-4bit \
  --quantize --q-bits 4 \
  --trust-remote-code

mlx_vlm.convert \
  --hf-path Inferact/MiniMax-M3-EAGLE3 \
  --mlx-path ~/MiniMax-M3-EAGLE3

mlx_vlm.generate \
  --model ~/MiniMax-M3-4bit \
  --draft-model ~/MiniMax-M3-EAGLE3 \
  --draft-kind eagle3 \
  --draft-block-size 3 \
  --prompt "Explain MiniMax Sparse Attention in one paragraph." \
  --max-tokens 256 --temperature 0

pip install -U 'mlx-vlm[ui]'

mlx_vlm.chat_ui --model mlx-community/Qwen2-VL-2B-Instruct-4bit

import mlx.core as mx
from mlx_vlm import load, generate
from mlx_vlm.prompt_utils import apply_chat_template
from mlx_vlm.utils import load_config

# Load the model
model_path = "mlx-community/Qwen2-VL-2B-Instruct-4bit"
model, processor = load(model_path)
config = load_config(model_path)

# Prepare input
image = ["http://images.cocodataset.org/val2017/000000039769.jpg"]
# image = [Image.open("...")] can also be used with PIL.Image.Image objects
prompt = "Describe this image."

# Apply chat template
formatted_prompt = apply_chat_template(
    processor, config, prompt, num_images=len(image)
)

# Generate output
output = generate(model, processor, formatted_prompt, image, verbose=False)
print(output)

from mlx_vlm import load, generate
from mlx_vlm.prompt_utils import apply_chat_template
from mlx_vlm.utils import load_config

# Load model with audio support
model_path = "mlx-community/gemma-3n-E2B-it-4bit"
model, processor = load(model_path)
config = model.config

# Prepare audio input
audio = ["/path/to/audio1.wav", "/path/to/audio2.mp3"]
prompt = "Describe what you hear in these audio files."

# Apply chat template with audio
formatted_prompt = apply_chat_template(
    processor, config, prompt, num_audios=len(audio)
)

# Generate output with audio
output = generate(model, processor, formatted_prompt, audio=audio, verbose=False)
print(output)

from mlx_vlm import load, generate
from mlx_vlm.prompt_utils import apply_chat_template
from mlx_vlm.utils import load_config

# Load multi-modal model
model_path = "mlx-community/gemma-3n-E2B-it-4bit"
model, processor = load(model_path)
config = model.config

# Prepare inputs
image = ["/path/to/image.jpg"]
audio = ["/path/to/audio.wav"]
prompt = ""

# Apply chat template
formatted_prompt = apply_chat_template(
    processor, config, prompt,
    num_images=len(image),
    num_audios=len(audio)
)

# Generate output
output = generate(model, processor, formatted_prompt, image, audio=audio, verbose=False)
print(output)

mlx_vlm.server --port 8080

# Preload a model at startup (Hugging Face repo or local path)
mlx_vlm.server --model <hf_repo_or_local_path>

# Preload separate model kinds at startup
mlx_vlm.server --model <language_model> \
  --image-model <image_generation_model> \
  --tts-model <text_to_speech_model> \
  --stt-model <speech_to_text_model>

# Preload a model with adapter
mlx_vlm.server --model <hf_repo_or_local_path> --adapter-path <adapter_path>

# With trust remote code enabled (required for some models)
mlx_vlm.server --trust-remote-code

# Enable thinking mode by default for requests that do not override it
mlx_vlm.server --model Qwen/Qwen3.5-4B --enable-thinking

# Configure thinking defaults at startup
mlx_vlm.server --model Qwen/Qwen3.5-4B \
  --enable-thinking \
  --thinking-budget 512 \
  --thinking-start-token "<think>" \
  --thinking-end-token "</think>"

# Require bearer authentication for API endpoints
mlx_vlm.server --api-key <secret-token>

MLX_TRUST_REMOTE_CODE=true mlx_vlm.server

mlx_vlm.server --port 8080 --model mlx-community/Qwen2.5-VL-3B-Instruct-4bit

curl http://localhost:8080/health
# {"status":"healthy","loaded_model":"...","apc_enabled":false}

from pathlib import Path

from mlx_vlm import load, stream_generate
from mlx_vlm.apc import APCManager, DiskBlockStore
from mlx_vlm.prompt_utils import apply_chat_template

model_id = "Qwen/Qwen3-VL-4B-Instruct"
model, processor = load(model_id)

disk = DiskBlockStore(
    Path("~/.cache/mlx-vlm/caching").expanduser(),
    namespace=model_id,
    max_bytes=3 * (1 << 30),  # 3 GB disk cap; use None for uncapped
)
apc = APCManager(num_blocks=4096, block_size=16, disk=disk)

document = Path("long_document.txt").read_text()

try:
    # First request computes the full prefix and stores reusable K/V blocks.
    prompt1 = apply_chat_template(
        processor,
        model.config,
        prompt=f"{document}\n\nSummarize the key decisions.",
        num_images=0,
    )
    for _ in stream_generate(
        model, processor, prompt1, max_tokens=128, temperature=0.0, apc_manager=apc
    ):
        pass

    # Second request shares the same document prefix and only prefills the suffix.
    prompt2 = apply_chat_template(
        processor,
        model.config,
        prompt=f"{document}\n\nList the open engineering risks.",
        num_images=0,
    )
    for chunk in stream_generate(
        model, processor, prompt2, max_tokens=128, temperature=0.0, apc_manager=apc
    ):
        print(chunk.text, end="", flush=True)

    print(apc.stats_snapshot())
finally:
    apc.close()

import os
import tempfile
import time
from pathlib import Path

from mlx_vlm import load, stream_generate
from mlx_vlm.apc import APCManager, DiskBlockStore
from mlx_vlm.prompt_utils import apply_chat_template

model_id = "Qwen/Qwen3-VL-4B-Instruct"
contexts = [8000, 20000, 50000, 100000]
disk_cap_gb = 0  # 0 means uncapped
shard_max_blocks = 256
context_sweep_max_tokens = 1  # one token is enough to measure prefill reuse

test_prompt_tokens = 8000
fill_prompts = 80
eviction_disk_cap_gb = 3.0

os.environ["APC_DISK_SHARD_MAX_BLOCKS"] = str(shard_max_blocks)

model, processor = load(model_id)
tokenizer = processor.tokenizer if hasattr(processor, "tokenizer") else processor


def disk_cap_bytes(gb: float):
    return None if gb <= 0 else int(gb * (1 << 30))


def make_context(target_tokens: int, seed: int = 0) -> str:
    line = (
        f"Document {seed}: APC benchmark content with deterministic facts, "
        "dates, identifiers, and repeated technical notes.\n"
    )
    line_tokens = max(1, len(tokenizer.encode(line, add_special_tokens=False)))
    text = line * max(1, target_tokens // line_tokens)
    while len(tokenizer.encode(text, add_special_tokens=False)) < target_tokens:
        text += line
    return text


def make_prompt(context: str, question: str) -> str:
    return apply_chat_template(
        processor,
        model.config,
        prompt=f"{context}\n\n{question}",
        num_images=0,
    )


def run_once(apc: APCManager, context: str, question: str, max_tokens: int = 32):
    prompt = make_prompt(context, question)
    apc.reset_stats()

    last = None
    output = []
    start = time.perf_counter()
    for chunk in stream_generate(
        model,
        processor,
        prompt,
        max_tokens=max_tokens,
        temperature=0.0,
        apc_manager=apc,
    ):
        output.append(chunk.text)
        last = chunk

    if last is None:
        raise RuntimeError("generation returned no chunks")

    return {
        "wall_s": time.perf_counter() - start,
        "prompt_tokens": last.prompt_tokens,
        "prompt_tps": last.prompt_tps,
        "generation_tps": last.generation_tps,
        "apc": apc.stats_snapshot(),
        "text": "".join(output).strip(),
    }


def print_result(label: str, result: dict) -> None:
    stats = result["apc"]
    print(
        f"{label:<12} "
        f"prompt_tokens={result['prompt_tokens']:>7} "
        f"prompt_tps={result['prompt_tps']:>8.1f} "
        f"gen_tps={result['generation_tps']:>7.1f} "
        f"matched={stats.get('matched_tokens', 0):>7} "
        f"disk_hits={stats.get('disk_hits', 0):>5} "
        f"disk_evictions={stats.get('disk_evictions', 0):>5}"
    )


def open_apc(cache_root: Path, namespace: str, disk_gb: float) -> APCManager:
    disk = DiskBlockStore(
        cache_root,
        namespace=namespace,
        max_bytes=disk_cap_bytes(disk_gb),
    )
    return APCManager(num_blocks=4096, block_size=16, disk=disk)


def run_context_sweep() -> None:
    print("cold / warm-memory / warm-disk")
    with tempfile.TemporaryDirectory() as tmp:
        cache_root = Path(tmp)
        for target_tokens in contexts:
            context = make_context(target_tokens)
            namespace = f"{model_id}-context-{target_tokens}"
            apc = open_apc(cache_root, namespace, disk_cap_gb)
            try:
                print(f"\ncontext ~= {target_tokens} text tokens")
                print_result(
                    "cold",
                    run_once(
                        apc,
                        context,
                        "Summarize the key decisions.",
                        max_tokens=context_sweep_max_tokens,
                    ),
                )
                print_result(
                    "warm-memory",
                    run_once(
                        apc,
                        context,
                        "List the open engineering risks.",
                        max_tokens=context_sweep_max_tokens,
                    ),
                )
            finally:
                # Closing waits for queued disk writes before reopening the disk tier.
                apc.close()

            apc = open_apc(cache_root, namespace, disk_cap_gb)
            try:
                print_result(
                    "warm-disk",
                    run_once(
                        apc,
                        context,
                        "Extract the implementation timeline.",
                        max_tokens=context_sweep_max_tokens,
                    ),
                )
            finally:
                apc.close()


def run_disk_eviction_workload() -> None:
    print("\ndisk eviction workload")
    with tempfile.TemporaryDirectory() as tmp:
        cache_root = Path(tmp)
        namespace = f"{model_id}-eviction"
        test_context = make_context(test_prompt_tokens, seed=0)

        apc = open_apc(cache_root, namespace, eviction_disk_cap_gb)
        try:
            print_result(
                "seed",
                run_once(apc, test_context, "Summarize the retained test prefix."),
            )
        finally:
            apc.close()

        apc = open_apc(cache_root, namespace, eviction_disk_cap_gb)
        try:
            for i in range(fill_prompts):
                fill_context = make_context(test_prompt_tokens, seed=i + 1)
                run_once(
                    apc,
                    fill_context,
                    f"Summarize filler document {i + 1}.",
                    max_tokens=1,
                )
                if (i + 1) % 10 == 0:
                    stats = apc.stats_snapshot()
                    print(
                        f"filled={i + 1:>3} "
                        f"disk_gb={stats.get('disk_bytes', 0) / (1 << 30):.2f} "
                        f"disk_evictions={stats.get('disk_evictions', 0)}"
                    )
        finally:
            apc.close()

        apc = open_apc(cache_root, namespace, eviction_disk_cap_gb)
        try:
            print_result(
                "post-fill",
                run_once(
                    apc,
                    test_context,
                    "Check whether the retained test prefix still restores.",
                ),
            )
        finally:
            apc.close()


run_context_sweep()
run_disk_eviction_workload()

APC_ENABLED=1 \
APC_NUM_BLOCKS=4096 \
mlx_vlm.server --model Qwen/Qwen3-VL-4B-Instruct --port 8080

APC_ENABLED=1 \
APC_NUM_BLOCKS=4096 \
mlx_vlm.server --model Qwen/Qwen3-VL-4B-Instruct --kv-bits 8 --port 8080

APC_ENABLED=1 \
APC_NUM_BLOCKS=4096 \
APC_DISK_PATH=~/.cache/mlx-vlm/caching \
APC_DISK_MAX_GB=3 \
APC_DISK_SHARD_MAX_BLOCKS=256 \
mlx_vlm.server --model Qwen/Qwen3-VL-4B-Instruct --port 8080

curl -X POST "http://localhost:8080/v1/chat/completions" \
  -H "Content-Type: application/json" \
  -H "X-APC-Tenant: demo" \
  -d '{
    "model": "Qwen/Qwen3-VL-4B-Instruct",
    "messages": [{
      "role": "user",
      "content": "Paste a long shared document here.\n\nNow answer question A."
    }],
    "max_tokens": 128
  }'

curl http://localhost:8080/v1/cache/stats
curl -X POST http://localhost:8080/v1/cache/reset

# Uniform 8-bit KV cache quantization
mlx_vlm.server --model google/gemma-4-26b-a4b-it --kv-bits 8

# TurboQuant 3.5-bit (3-bit keys + 4-bit values)
mlx_vlm.server --model google/gemma-4-26b-a4b-it --kv-bits 3.5 --kv-quant-scheme turboquant

curl -X POST "http://localhost:8080/v1/chat/completions" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "mlx-community/Qwen2-VL-2B-Instruct-4bit",
    "messages": [{"role":"user","content":"Say hi in 3 words."}],
    "max_tokens": 8,
    "logprobs": true,
    "top_logprobs": 3
  }'

mlx_vlm.server --model mlx-community/Qwen2-VL-2B-Instruct-4bit --top-logprobs-k 5
# or
TOP_LOGPROBS_K=5 mlx_vlm.server --model mlx-community/Qwen2-VL-2B-Instruct-4bit

from typing import Literal

from pydantic import BaseModel, ConfigDict, Field


class AnimalResult(BaseModel):
    model_config = ConfigDict(extra="forbid")

    animal: Literal["dog", "cat", "bird", "unknown"]
    species: str = Field(max_length=60)
    description: str = Field(max_length=200)


schema = AnimalResult.model_json_schema()

from openai import OpenAI

client = OpenAI(base_url="http://localhost:8080/v1", api_key="not-needed")

response = client.chat.completions.create(
    model="mlx-community/Qwen3.5-4B-MLX-4bit",
    messages=[
        {"role": "user", "content": "Return a dog object."},
    ],
    response_format={
        "type": "json_schema",
        "json_schema": {
            "name": "AnimalResult",
            "strict": True,
            "schema": schema,
        },
    },
)

result = AnimalResult.model_validate_json(response.choices[0].message.content)
print(result)

animal='dog' species='Canis lupus familiaris' description='A domesticated canine known for companionship and loyalty.'

curl -X POST "http://localhost:8080/v1/chat/completions" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "mlx-community/Qwen3.5-4B-MLX-4bit",
    "messages": [{
      "role": "user",
      "content": [
        {"type": "text", "text": "Identify the main animal in this image."},
        {"type": "image_url", "image_url": {"url": "/path/to/image.jpg"}}
      ]
    }],
    "response_format": {
      "type": "json_schema",
      "json_schema": {
        "name": "AnimalResult",
        "strict": true,
        "schema": {
          "type": "object",
          "properties": {
            "animal": {"type": "string", "enum": ["dog", "cat", "bird", "unknown"]},
            "species": {"type": "string", "maxLength": 60},
            "description": {"type": "string", "maxLength": 200}
          },
          "required": ["animal", "species", "description"],
          "additionalProperties": false
        }
      }
    },
    "max_tokens": 256
  }'

curl "http://localhost:8080/models"

curl -X POST "http://localhost:8080/v1/embeddings" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "sentence-transformers/all-MiniLM-L6-v2",
    "input": ["The quick brown fox.", "A fast auburn fox."]
  }'

curl -X POST "http://localhost:8080/chat/completions" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "mlx-community/Qwen2-VL-2B-Instruct-4bit",
    "messages": [
      {
        "role": "user",
        "content": "Hello, how are you"
      }
    ],
    "stream": true,
    "max_tokens": 100
  }'

curl -X POST "http://localhost:8080/chat/completions" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "mlx-community/Qwen2.5-VL-32B-Instruct-8bit",
    "messages":
    [
      {
        "role": "system",
        "content": "You are a helpful assistant."
      },
      {
        "role": "user",
        "content": [
          {
            "type": "text",
            "text": "This is today's chart for energy demand in California. Can you provide an analysis of the chart and comment on the implications for renewable energy in California?"
          },
          {
            "type": "input_image",
            "image_url": "/path/to/repo/examples/images/renewables_california.png"
          }
        ]
      }
    ],
    "stream": true,
    "max_tokens": 1000
  }'

curl -X POST "http://localhost:8080/chat/completions" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "mlx-community/gemma-3n-E2B-it-4bit",
    "messages": [
      {
        "role": "user",
        "content": [
          { "type": "text", "text": "Describe what you hear in these audio files" },
          { "type": "input_audio", "input_audio": "/path/to/audio1.wav" },
          { "type": "input_audio", "input_audio": "https://example.com/audio2.mp3" }
        ]
      }
    ],
    "stream": true,
    "max_tokens": 500
  }'

curl -X POST "http://localhost:8080/v1/audio/speech" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "mlx-community/pocket-tts",
    "input": "Hello from MLX VLM.",
    "voice": "fantine",
    "response_format": "mp3"
  }' \
  --output speech.mp3

curl -X POST "http://localhost:8080/v1/audio/transcriptions" \
  -F model=mlx-community/parakeet-tdt-0.6b-v3 \
  -F file=@/path/to/audio.mp3 \
  -F response_format=json

curl -X POST "http://localhost:8080/chat/completions" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "mlx-community/gemma-3n-E2B-it-4bit",
    "messages": [
      {
        "role": "user",
        "content": [
          {"type": "input_image", "image_url": "/path/to/image.jpg"},
          {"type": "input_audio", "input_audio": "/path/to/audio.wav"}
        ]
      }
    ],
    "max_tokens": 100
  }'

curl -X POST "http://localhost:8080/responses" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "mlx-community/Qwen2-VL-2B-Instruct-4bit",
    "messages": [
      {
        "role": "user",
        "content": [
          {"type": "input_text", "text": "What is in this image?"},
          {"type": "input_image", "image_url": "/path/to/image.jpg"}
        ]
      }
    ],
    "max_tokens": 100
  }'

{
  "quantization": {
    "group_size": 64,
    "bits": 1,
    "mode": "affine"
  }
}

from mlx_vlm import generate, load

model, processor = load("path/to/1bit-model")
result = generate(model, processor, "Describe this image", image=["image.jpg"])

mlx_vlm.generate --model /path/to/mxfp8-model --prompt "Describe this image" --image /path/to/image.jpg -qa

from mlx_vlm import load, generate

# Load with activation quantization enabled
model, processor = load(
    "path/to/mxfp8-quantized-model",
    quantize_activations=True
)

# Generate as usual
output = generate(model, processor, "Describe this image", image=["image.jpg"])

from mlx_vlm import load, generate
from mlx_vlm.prompt_utils import apply_chat_template
from mlx_vlm.utils import load_config

model_path = "mlx-community/Qwen2-VL-2B-Instruct-4bit"
model, processor = load(model_path)
config = model.config

images = ["path/to/image1.jpg", "path/to/image2.jpg"]
prompt = "Compare these two images."

formatted_prompt = apply_chat_template(
    processor, config, prompt, num_images=len(images)
)

output = generate(model, processor, formatted_prompt, images, verbose=False)
print(output)

mlx_vlm.generate --model mlx-community/Qwen2-VL-2B-Instruct-4bit --max-tokens 100 --prompt "Compare these images" --image path/to/image1.jpg path/to/image2.jpg

mlx_vlm.generate --model mlx-community/Qwen2-VL-2B-Instruct-4bit --max-tokens 100 --prompt "Describe this video" --video path/to/video.mp4 --fps 1.0

# Gradio chat UI
python -m mlx_vlm.chat_ui --model google/gemma-4-26b-a4b-it

# Interactive chat with Rich UI (load images with /image command)
python -m mlx_vlm.chat --model google/gemma-4-26b-a4b-it

# Inline chat mode
python -m mlx_vlm.generate \
  --model google/gemma-4-26b-a4b-it \
  --image path/to/image.jpg \
  --chat \
  --max-tokens 200

from mlx_vlm import load, stream_generate, VisionFeatureCache
from mlx_vlm.prompt_utils import apply_chat_template

model, processor = load("google/gemma-4-26b-a4b-it")
cache = VisionFeatureCache()

image = "path/to/image.jpg"

# Turn 1 -- cache miss, encodes image
prompt1 = apply_chat_template(processor, model.config, "Describe this image.", num_images=1)
for chunk in stream_generate(model, processor, prompt1, image=[image],
                              max_tokens=200, vision_cache=cache):
    print(chunk.text, end="")

# Turn 2 -- cache hit, skips vision encoder
prompt2 = apply_chat_template(processor, model.config, "What colors do you see?", num_images=1)
for chunk in stream_generate(model, processor, prompt2, image=[image],
                              max_tokens=200, vision_cache=cache):
    print(chunk.text, end="")

mlx_vlm.server --model google/gemma-4-26b-a4b-it

# 3.5-bit KV cache quantization (3-bit keys + 4-bit values)
mlx_vlm generate \
  --model mlx-community/Qwen3.5-4B-4bit \
  --kv-bits 3.5 \
  --kv-quant-scheme turboquant \
  --prompt "Your long prompt here..."

from mlx_vlm import generate

result = generate(
    model, processor, prompt,
    kv_bits=3.5,
    kv_quant_scheme="turboquant",
    max_tokens=256,
)

# Server with TurboQuant
mlx_vlm server \
  --model google/gemma-4-26b-a4b-it \
  --kv-bits 3.5 \
  --kv-quant-scheme turboquant

pip install "mlx-vlm[train]"