{ "openapi": "3.1.0", "info": { "title": "mashbox server", "version": "0.1.13", "description": "The HTTP server `mashbox run ` starts on 127.0.0.1 (reach it from your machine with an SSH tunnel). It speaks the OpenAI Chat Completions, Completions and Responses APIs and the Anthropic Messages API, so OpenAI and Anthropic clients and coding agents work against it unchanged. On 127.0.0.1 no API key is checked and any key a client sends is ignored; with `--api-key-file` or `MASHBOX_API_KEY`, and always when `--host` is not loopback, `/v1` routes, `/metrics` and `/openapi.json` need a key (`Authorization: Bearer ` or `x-api-key: `; 401 without). POST bodies must be `Content-Type: application/json` (415 otherwise), and a request from a web page of another origin, or naming the server by a host name it does not answer to, gets 403. Fields a route does not list are accepted and ignored." }, "servers": [ { "url": "http://127.0.0.1:8000", "description": "`mashbox run ` (default `--port 8000`)" } ], "tags": [ { "name": "openai", "description": "OpenAI-compatible routes." }, { "name": "anthropic", "description": "Anthropic-compatible routes." }, { "name": "mashbox", "description": "Health, metrics and this description." } ], "paths": { "/v1/health": { "get": { "operationId": "health", "summary": "Health", "description": "200 when the model is loaded and the server takes requests, 503 while it loads. GET /health is the same.", "tags": [ "mashbox" ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Health" } } } }, "503": { "description": "The model is still loading.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Health" } } } } } }, "head": { "operationId": "healthHead", "summary": "Ready check without a body", "description": "200 when ready, 503 while loading, no body. The server answers HEAD the same way on every path.", "tags": [ "mashbox" ], "responses": { "200": { "description": "Ready." }, "503": { "description": "Loading." } } } }, "/health": { "get": { "operationId": "healthShort", "summary": "Health (short path)", "description": "Same as GET /v1/health.", "tags": [ "mashbox" ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Health" } } } }, "503": { "description": "The model is still loading.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Health" } } } } } } }, "/": { "get": { "operationId": "root", "summary": "Health, or the chat page", "description": "Same as GET /v1/health. With `mashbox run --ui` it is the MASHBOX Chat page instead.", "tags": [ "mashbox" ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Health" } } } }, "503": { "description": "The model is still loading.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Health" } } } } } } }, "/v1/models": { "get": { "operationId": "listModels", "summary": "List models", "description": "The one model this server runs.", "tags": [ "openai" ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ModelList" } } } } } } }, "/v1/models/{model}": { "get": { "operationId": "getModel", "summary": "Get a model", "description": "The model, when {model} is its id; 404 otherwise.", "tags": [ "openai" ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Model" } } } }, "404": { "description": "Another model id.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "parameters": [ { "name": "model", "in": "path", "required": true, "schema": { "type": "string" } } ] } }, "/v1/chat/completions": { "post": { "operationId": "createChatCompletion", "summary": "Chat completion", "description": "OpenAI Chat Completions with function tools, thinking and streaming. Coding agents such as Pi use it.", "tags": [ "openai" ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ChatCompletion" } }, "text/event-stream": { "schema": { "$ref": "#/components/schemas/ChatCompletionChunk" } } } }, "400": { "description": "Bad request.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "A model id other than the one this server serves (GET /v1/models), code model_not_found. Leaving model out is accepted.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "Too many requests are waiting for a slot: 32 of yours (per tenant: keys the key file names with one tenant share it, a key without a name is its own tenant; without keys, 32 in all), or the server's queue is full (256 waiting in all). Retry after the Retry-After header's seconds.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "500": { "description": "The request failed on the device: its one retry failed too, or it failed after its stream had sent output. A request whose client has received nothing is sent again (once, from the start): not streamed, until its response is written; streamed, until its first output is sent. A stream that has sent output is never retried by the server, since a second run could repeat what the client already has; it ends with the route's error event instead, carrying the response id as request_id.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "The model is still loading. Retry after the Retry-After header's seconds.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ChatCompletionRequest" }, "example": { "model": "qwen3-8-27b", "messages": [ { "role": "user", "content": "Name three planets." } ], "max_tokens": 64 } } } } } }, "/v1/completions": { "post": { "operationId": "createCompletion", "summary": "Text completion", "description": "Continue a plain prompt, no chat template.", "tags": [ "openai" ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Completion" } }, "text/event-stream": { "schema": { "$ref": "#/components/schemas/Completion" } } } }, "400": { "description": "Bad request.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "A model id other than the one this server serves (GET /v1/models), code model_not_found. Leaving model out is accepted.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "Too many requests are waiting for a slot: 32 of yours (per tenant: keys the key file names with one tenant share it, a key without a name is its own tenant; without keys, 32 in all), or the server's queue is full (256 waiting in all). Retry after the Retry-After header's seconds.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "500": { "description": "The request failed on the device: its one retry failed too, or it failed after its stream had sent output. A request whose client has received nothing is sent again (once, from the start): not streamed, until its response is written; streamed, until its first output is sent. A stream that has sent output is never retried by the server, since a second run could repeat what the client already has; it ends with the route's error event instead, carrying the response id as request_id.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "The model is still loading. Retry after the Retry-After header's seconds.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CompletionRequest" }, "example": { "prompt": "The three largest planets are", "max_tokens": 32 } } } } } }, "/v1/messages": { "post": { "operationId": "createMessage", "summary": "Messages (Anthropic)", "description": "Anthropic Messages with client tools, thinking and streaming, for Claude Code. The ?beta=true query Claude Code adds is accepted.", "tags": [ "anthropic" ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Message" } }, "text/event-stream": { "schema": { "$ref": "#/components/schemas/MessageStreamEvent" } } } }, "400": { "description": "Bad request.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnthropicError" } } } }, "429": { "description": "Too many requests are waiting for a slot: 32 of yours (per tenant: keys the key file names with one tenant share it, a key without a name is its own tenant; without keys, 32 in all), or the server's queue is full (256 waiting in all). Retry after the Retry-After header's seconds.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnthropicError" } } } }, "500": { "description": "The request failed on the device: its one retry failed too, or it failed after its stream had sent output. A request whose client has received nothing is sent again (once, from the start): not streamed, until its response is written; streamed, until its first output is sent. A stream that has sent output is never retried by the server, since a second run could repeat what the client already has; it ends with the route's error event instead, carrying the response id as request_id.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnthropicError" } } } }, "529": { "description": "The model is still loading, or its worker is starting again after a fault, or the server's queue is full (256 waiting in all): overloaded_error. Retry after the Retry-After header's seconds (x-should-retry: true while not ready).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnthropicError" } } } } }, "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MessagesRequest" }, "example": { "model": "qwen3-8-27b", "max_tokens": 64, "messages": [ { "role": "user", "content": "Name three planets." } ] } } } } } }, "/v1/messages/count_tokens": { "post": { "operationId": "countTokens", "summary": "Count tokens", "description": "Prompt tokens a Messages request would use, after the chat template.", "tags": [ "anthropic" ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TokenCount" } } } }, "400": { "description": "Bad request.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnthropicError" } } } }, "529": { "description": "The model is still loading, or its worker is starting again after a fault, or the server's queue is full (256 waiting in all): overloaded_error. Retry after the Retry-After header's seconds (x-should-retry: true while not ready).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnthropicError" } } } } }, "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CountTokensRequest" }, "example": { "model": "qwen3-8-27b", "messages": [ { "role": "user", "content": "Name three planets." } ] } } } } } }, "/v1/responses": { "post": { "operationId": "createResponse", "summary": "Responses (OpenAI)", "description": "OpenAI Responses with function, custom and namespace tools, reasoning and streaming, for Codex.", "tags": [ "openai" ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Response" } }, "text/event-stream": { "schema": { "$ref": "#/components/schemas/ResponseStreamEvent" } } } }, "400": { "description": "Bad request.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "A model id other than the one this server serves (GET /v1/models), code model_not_found. Leaving model out is accepted.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "Too many requests are waiting for a slot: 32 of yours (per tenant: keys the key file names with one tenant share it, a key without a name is its own tenant; without keys, 32 in all), or the server's queue is full (256 waiting in all). Retry after the Retry-After header's seconds.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "500": { "description": "The request failed on the device: its one retry failed too, or it failed after its stream had sent output. A request whose client has received nothing is sent again (once, from the start): not streamed, until its response is written; streamed, until its first output is sent. A stream that has sent output is never retried by the server, since a second run could repeat what the client already has; it ends with the route's error event instead, carrying the response id as request_id.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "The model is still loading. Retry after the Retry-After header's seconds.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ResponsesRequest" }, "example": { "model": "qwen3-8-27b", "input": "Name three planets.", "max_output_tokens": 64 } } } } } }, "/metrics": { "get": { "operationId": "metrics", "summary": "Metrics (Prometheus text)", "description": "What the server does, in the Prometheus text format: requests running and waiting, KV cache use, the cache layout (one slot holds one whole context: block_size = the context length, num_blocks = the slots), requests answered by finish reason, prompt and generated tokens, prefix cache lookups and hits (tokens), preemptions (never), worker restarts, programs compiled after ready, and histograms of time to first token, queue time, end-to-end time and time per output token. Every name starts with `mashbox:`. No label names a user, an API key or a session.", "tags": [ "mashbox" ], "responses": { "200": { "description": "Prometheus text exposition format.", "content": { "text/plain": { "schema": { "type": "string" } } } } } } }, "/openapi.json": { "get": { "operationId": "openapi", "summary": "This description", "description": "The OpenAPI 3.1 description of this server, with the running model's id.", "tags": [ "mashbox" ], "responses": { "200": { "description": "OpenAPI 3.1 document.", "content": { "application/json": { "schema": { "type": "object" } } } } } } } }, "components": { "schemas": { "Health": { "type": "object", "properties": { "status": { "type": "string", "description": "ok when the model is loaded, loading before (and while its worker restarts).", "enum": [ "ok", "loading" ] }, "live_rows": { "type": [ "integer", "null" ], "description": "Requests being answered now." }, "waiting": { "type": "integer", "description": "Requests waiting for a slot." }, "cache_bytes": { "type": "integer", "description": "Bytes the prompt cache holds in host memory: host_cache_bytes plus snapshot_bytes, each copy counted once." }, "host_cache_bytes": { "type": "integer", "description": "Bytes of the host memory prompt cache, each copy counted once." }, "snapshot_bytes": { "type": "integer", "description": "Bytes of saved slot state points that host_cache_bytes does not already count." }, "host_cache_cap_bytes": { "type": "integer", "description": "Most bytes the host memory prompt cache holds (0 = off)." }, "host_cache_evictions": { "type": "integer", "description": "Prompt cache entries removed from host memory to stay under that cap." }, "cache_namespace": { "type": [ "string", "null" ], "description": "Id of what cached prompts are valid for: model, weights, tokenizer, chat template and cache policy." }, "deployment_id": { "type": [ "string", "null" ], "description": "Id of the compiled programs this server runs (null until set)." }, "component_set_sha": { "type": [ "string", "null" ], "description": "Id of the model's component set (null until set)." }, "certification": { "type": "string", "description": "Whether this server's build passed the certification gate on its chip.", "enum": [ "certified", "not_certified" ] }, "compiles_after_ready": { "type": [ "integer", "null" ], "description": "Programs compiled since the server reported ready, in every worker process it has run (0 when the start-up warm-up compiled every program the requests ran; null before ready or where this process does not count them)." }, "routed_experts": { "type": [ "object", "null" ], "description": "Mixture-of-experts models: the path the routed experts run on, read where the engine runs. path: kernel (the kernel package's routed-expert kernel at every call site), exact (the exact path) or mixed; kernel: that kernel's id, version and package (null when the package has none); sites: the routed-op call sites traced on each path; exact_why: why any site ran exact. Null for a dense model and before the first program is traced.", "properties": { "path": { "type": "string", "description": "kernel, exact or mixed.", "enum": [ "kernel", "exact", "mixed" ] }, "kernel": { "type": [ "object", "null" ], "description": "id, version, package." }, "sites": { "type": "object", "properties": { "kernel": { "type": "integer", "description": "Call sites on the kernel." }, "exact": { "type": "integer", "description": "Call sites on the exact path." } } }, "exact_why": { "type": "array", "description": "Why sites ran exact.", "items": { "type": "string" } } } }, "worker_pid": { "type": "integer", "description": "Process id of the model's worker, when the engine runs in a worker process." }, "worker_restarts": { "type": "integer", "description": "Times the worker was started again after a fault." } }, "required": [ "status" ] }, "Model": { "type": "object", "properties": { "id": { "type": "string", "description": "Model id, as in models.json." }, "object": { "type": "string", "description": "model", "const": "model" }, "owned_by": { "type": "string", "description": "mashbox-labs" }, "created": { "type": "integer", "description": "Always 0." }, "max_model_len": { "type": "integer", "description": "Context length the server runs (--ctx)." }, "weights": { "type": "string", "description": "Weights file loaded." }, "gb": { "type": "number", "description": "Weights in device memory, GB." }, "bpw": { "type": "number", "description": "Bits per weight of the weights file." }, "prefix_cache": { "type": "boolean", "description": "Whether the server reuses cached prompt starts." }, "slots": { "type": "integer", "description": "Requests decoded together (--slots)." }, "cache_ram_mib": { "type": "integer", "description": "Host memory for saved prompt states (--cache-ram)." } }, "required": [ "id", "object", "owned_by", "created", "max_model_len" ] }, "ModelList": { "type": "object", "properties": { "object": { "type": "string", "description": "list", "const": "list" }, "data": { "type": "array", "items": { "$ref": "#/components/schemas/Model" } } }, "required": [ "object", "data" ] }, "Error": { "type": "object", "properties": { "error": { "type": "object", "properties": { "message": { "type": "string", "description": "What went wrong." }, "type": { "type": "string", "description": "invalid_request_error, unavailable or server_error." } }, "required": [ "message", "type" ] } }, "required": [ "error" ], "description": "OpenAI error shape (all routes except /v1/messages*)." }, "AnthropicError": { "type": "object", "properties": { "type": { "type": "string", "description": "error", "const": "error" }, "error": { "type": "object", "properties": { "type": { "type": "string", "description": "invalid_request_error, overloaded_error or api_error." }, "message": { "type": "string", "description": "What went wrong." } }, "required": [ "type", "message" ] } }, "required": [ "type", "error" ], "description": "Anthropic error shape (/v1/messages and /v1/messages/count_tokens)." }, "ChatMessage": { "type": "object", "properties": { "role": { "type": "string", "description": "system and developer join the system prompt.", "enum": [ "system", "developer", "user", "assistant", "tool", "function" ] }, "content": { "type": [ "string", "array", "null" ], "description": "Text, or a list of text parts. Text parts only: image, audio and file parts are refused with 400 (text-only model)." }, "tool_calls": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "description": "Call id." }, "type": { "type": "string", "description": "function", "enum": [ "function" ] }, "function": { "type": "object", "properties": { "name": { "type": "string", "description": "Tool name." }, "arguments": { "type": "string", "description": "Arguments as a JSON string." } }, "required": [ "name", "arguments" ] } }, "required": [ "id", "type", "function" ] }, "description": "assistant: the calls it made." }, "tool_call_id": { "type": "string", "description": "tool: the call this answers." }, "reasoning_content": { "type": "string", "description": "assistant: its earlier thinking." }, "name": { "type": "string", "description": "Accepted and ignored." } }, "required": [ "role" ] }, "ChatCompletionRequest": { "type": "object", "properties": { "model": { "type": "string", "description": "Accepted for compatibility; the server answers with the one model it runs." }, "messages": { "type": "array", "items": { "$ref": "#/components/schemas/ChatMessage" }, "description": "The conversation." }, "tools": { "type": "array", "items": { "type": "object", "properties": { "type": { "type": "string", "description": "function", "enum": [ "function" ] }, "function": { "type": "object", "properties": { "name": { "type": "string", "description": "Tool name." }, "description": { "type": "string", "description": "What the tool does." }, "parameters": { "type": "object", "description": "JSON Schema of the arguments." } }, "required": [ "name" ] } }, "required": [ "function" ] }, "description": "Function tools the model may call." }, "tool_choice": { "type": [ "string", "object" ], "description": "auto (default), none, required, or {\"type\": \"function\", \"function\": {\"name\": ...}}." }, "stream": { "type": "boolean", "description": "Stream the answer as server-sent events (see the 200 text/event-stream response). A client that reads nothing from the stream for 30 s is disconnected and its answer stops.", "default": false }, "stream_options": { "type": "object", "properties": { "include_usage": { "type": "boolean", "description": "Send a last chunk with usage." } } }, "chat_template_kwargs": { "type": "object", "properties": { "enable_thinking": { "type": "boolean", "description": "Think before answering (Qwen3 thinking mode).", "default": false } }, "description": "Chat template options." }, "enable_thinking": { "type": "boolean", "description": "Same as chat_template_kwargs.enable_thinking." }, "merge": { "type": "boolean", "description": "Accepted and ignored: every request joins the running batch at the next device call, and no request field changes how other requests are scheduled.", "default": true }, "prompt_cache_key": { "type": "string", "description": "Tags the request's session in the server log (as the X-Session-Id header does)." }, "store": { "type": "boolean", "description": "Accepted and ignored." }, "logprobs": { "type": "boolean", "description": "Not supported (log probabilities): a request that sets it gets 400 naming the field." }, "top_logprobs": { "type": "integer", "description": "Not supported (log probabilities): a request that sets it gets 400 naming the field." }, "response_format": { "type": "object", "description": "Only {\"type\": \"text\"}. Not supported: json_object and json_schema (structured output), answered with 400." }, "max_tokens": { "type": "integer", "description": "Most new tokens (>= 1).", "minimum": 1 }, "max_completion_tokens": { "type": "integer", "description": "Most new tokens (>= 1); max_tokens is the same. Default: up to the end of the context (--ctx).", "minimum": 1 }, "temperature": { "type": "number", "description": "0 = greedy (default). Up to 4.", "minimum": 0, "maximum": 4, "default": 0.0 }, "top_p": { "type": "number", "description": "Nucleus sampling, in (0, 1].", "maximum": 1, "default": 1.0 }, "top_k": { "type": "integer", "description": "Keep the k most likely tokens; 0 = off.", "minimum": 0, "default": 0 }, "min_p": { "type": "number", "description": "Drop tokens below min_p times the top probability, in [0, 1).", "minimum": 0, "default": 0.0 }, "seed": { "type": "integer", "description": "Sampling seed; a random one when absent." }, "stop": { "type": [ "string", "array" ], "description": "A string or a list of non-empty strings; the text ends before the first one." }, "stop_sequences": { "type": "array", "description": "Same as stop (Anthropic's name), used when stop is absent.", "items": { "type": "string" } }, "repetition_penalty": { "type": "number", "description": "> 0; 1 = off.", "exclusiveMinimum": 0, "default": 1.0 }, "frequency_penalty": { "type": "number", "description": "-2 to 2; 0 = off.", "minimum": -2, "maximum": 2, "default": 0.0 }, "presence_penalty": { "type": "number", "description": "-2 to 2; 0 = off.", "minimum": -2, "maximum": 2, "default": 0.0 }, "n": { "type": "integer", "description": "Must be 1.", "enum": [ 1 ] } }, "required": [ "messages" ] }, "ChatCompletion": { "type": "object", "properties": { "id": { "type": "string", "description": "chatcmpl-..." }, "object": { "type": "string", "description": "chat.completion", "const": "chat.completion" }, "created": { "type": "integer", "description": "Unix time." }, "model": { "type": "string", "description": "Model id." }, "choices": { "type": "array", "items": { "type": "object", "properties": { "index": { "type": "integer", "description": "0." }, "message": { "type": "object", "properties": { "role": { "type": "string", "description": "assistant" }, "content": { "type": [ "string", "null" ], "description": "The answer (null when it only calls tools)." }, "reasoning_content": { "type": "string", "description": "Thinking, when thinking is on." }, "tool_calls": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "description": "Call id." }, "type": { "type": "string", "description": "function", "enum": [ "function" ] }, "function": { "type": "object", "properties": { "name": { "type": "string", "description": "Tool name." }, "arguments": { "type": "string", "description": "Arguments as a JSON string." } }, "required": [ "name", "arguments" ] } }, "required": [ "id", "type", "function" ] } } }, "required": [ "role", "content" ] }, "finish_reason": { "type": "string", "description": "stop, length or tool_calls.", "enum": [ "stop", "length", "tool_calls" ] } }, "required": [ "index", "message", "finish_reason" ] } }, "usage": { "$ref": "#/components/schemas/ChatUsage" }, "timings": { "$ref": "#/components/schemas/ChatTimings" } }, "required": [ "id", "object", "created", "model", "choices", "usage" ] }, "ChatTimings": { "type": "object", "properties": { "prompt_n": { "type": "integer", "description": "Prompt tokens prefilled (not served from the prefix cache)." }, "prompt_ms": { "type": "number", "description": "Milliseconds from the start of the run to the first new token." }, "prompt_per_second": { "type": [ "number", "null" ], "description": "prompt_n per second of prompt_ms." }, "predicted_n": { "type": "integer", "description": "New tokens." }, "predicted_ms": { "type": "number", "description": "Milliseconds from the first to the last new token." }, "predicted_per_second": { "type": [ "number", "null" ], "description": "Decode speed: new tokens after the first, per second of predicted_ms." }, "cache_n": { "type": "integer", "description": "Prompt tokens served from the prefix cache." }, "queue_ms": { "type": "number", "description": "Milliseconds the request waited before it ran." } }, "required": [ "prompt_n", "prompt_ms", "predicted_n", "predicted_ms", "cache_n", "queue_ms" ], "description": "Server timings, the fields of llama-server's timings block." }, "ChatUsage": { "type": "object", "properties": { "prompt_tokens": { "type": "integer", "description": "Prompt tokens." }, "completion_tokens": { "type": "integer", "description": "New tokens." }, "total_tokens": { "type": "integer", "description": "Sum." }, "prompt_tokens_details": { "type": "object", "properties": { "cached_tokens": { "type": "integer", "description": "Prompt tokens served from the prefix cache." } } } }, "required": [ "prompt_tokens", "completion_tokens", "total_tokens" ] }, "ChatCompletionChunk": { "type": "object", "properties": { "id": { "type": "string", "description": "Same id on every chunk." }, "object": { "type": "string", "description": "chat.completion.chunk", "const": "chat.completion.chunk" }, "created": { "type": "integer", "description": "Unix time." }, "model": { "type": "string", "description": "Model id." }, "choices": { "type": "array", "items": { "type": "object", "properties": { "index": { "type": "integer", "description": "0." }, "delta": { "type": "object", "properties": { "role": { "type": "string", "description": "assistant, first chunk only." }, "content": { "type": "string", "description": "New text." }, "reasoning_content": { "type": "string", "description": "New thinking." }, "tool_calls": { "type": "array", "items": { "type": "object", "properties": { "index": { "type": "integer", "description": "Call number." }, "id": { "type": "string", "description": "Call id." }, "type": { "type": "string", "description": "function" }, "function": { "type": "object", "properties": { "name": { "type": "string", "description": "Tool name." }, "arguments": { "type": "string", "description": "Whole arguments, JSON." } } } } } } } }, "finish_reason": { "type": [ "string", "null" ], "description": "null until the last chunk." } }, "required": [ "index", "delta" ] } }, "usage": { "$ref": "#/components/schemas/ChatUsage" }, "timings": { "$ref": "#/components/schemas/ChatTimings" } }, "required": [ "id", "object", "created", "model", "choices" ], "description": "One `data:` line. The stream ends with `data: [DONE]`. With stream_options.include_usage the last chunk has empty choices and usage. The chunk with the finish_reason, and the usage chunk, carry timings." }, "CompletionRequest": { "type": "object", "properties": { "model": { "type": "string", "description": "Accepted for compatibility; the server answers with the one model it runs." }, "prompt": { "type": [ "string", "array" ], "description": "The text to continue (a string, or a list of one string)." }, "stream": { "type": "boolean", "description": "Stream the answer as server-sent events (see the 200 text/event-stream response). A client that reads nothing from the stream for 30 s is disconnected and its answer stops.", "default": false }, "merge": { "type": "boolean", "description": "Accepted and ignored: every request joins the running batch at the next device call, and no request field changes how other requests are scheduled.", "default": true }, "prompt_cache_key": { "type": "string", "description": "Tags the request's session in the server log (as the X-Session-Id header does)." }, "logprobs": { "type": "integer", "description": "Not supported (log probabilities): a request that sets it gets 400 naming the field." }, "max_tokens": { "type": "integer", "description": "Most new tokens (>= 1).", "minimum": 1 }, "max_completion_tokens": { "type": "integer", "description": "Same as max_tokens (OpenAI's newer name); wins when both are set.", "minimum": 1 }, "temperature": { "type": "number", "description": "0 = greedy (default). Up to 4.", "minimum": 0, "maximum": 4, "default": 0.0 }, "top_p": { "type": "number", "description": "Nucleus sampling, in (0, 1].", "maximum": 1, "default": 1.0 }, "top_k": { "type": "integer", "description": "Keep the k most likely tokens; 0 = off.", "minimum": 0, "default": 0 }, "min_p": { "type": "number", "description": "Drop tokens below min_p times the top probability, in [0, 1).", "minimum": 0, "default": 0.0 }, "seed": { "type": "integer", "description": "Sampling seed; a random one when absent." }, "stop": { "type": [ "string", "array" ], "description": "A string or a list of non-empty strings; the text ends before the first one." }, "stop_sequences": { "type": "array", "description": "Same as stop (Anthropic's name), used when stop is absent.", "items": { "type": "string" } }, "repetition_penalty": { "type": "number", "description": "> 0; 1 = off.", "exclusiveMinimum": 0, "default": 1.0 }, "frequency_penalty": { "type": "number", "description": "-2 to 2; 0 = off.", "minimum": -2, "maximum": 2, "default": 0.0 }, "presence_penalty": { "type": "number", "description": "-2 to 2; 0 = off.", "minimum": -2, "maximum": 2, "default": 0.0 }, "n": { "type": "integer", "description": "Must be 1.", "enum": [ 1 ] } }, "required": [ "prompt" ] }, "Completion": { "type": "object", "properties": { "id": { "type": "string", "description": "cmpl-..." }, "object": { "type": "string", "description": "text_completion", "const": "text_completion" }, "created": { "type": "integer", "description": "Unix time." }, "model": { "type": "string", "description": "Model id." }, "choices": { "type": "array", "items": { "type": "object", "properties": { "index": { "type": "integer", "description": "0." }, "text": { "type": "string", "description": "The continuation (in a stream: new text)." }, "finish_reason": { "type": [ "string", "null" ], "description": "stop or length; null until the last chunk." } }, "required": [ "index", "text", "finish_reason" ] } }, "usage": { "type": "object", "properties": { "prompt_tokens": { "type": "integer", "description": "Prompt tokens." }, "completion_tokens": { "type": "integer", "description": "New tokens." }, "total_tokens": { "type": "integer", "description": "Sum." } } }, "system_fingerprint": { "type": "string", "description": "seed-." } }, "required": [ "id", "object", "created", "model", "choices" ], "description": "Whole answer, or one `data:` line of a stream (no usage); the stream ends with `data: [DONE]`." }, "MessagesRequest": { "type": "object", "properties": { "model": { "type": "string", "description": "Accepted for compatibility; the server answers with the one model it runs." }, "messages": { "type": "array", "items": { "type": "object", "properties": { "role": { "type": "string", "description": "user or assistant (system also accepted).", "enum": [ "user", "assistant", "system", "developer" ] }, "content": { "type": [ "string", "array" ], "description": "Text, or content blocks: text, thinking, tool_use, tool_result. Text parts only: image, audio and file parts are refused with 400 (text-only model)." } }, "required": [ "role", "content" ] }, "description": "The conversation." }, "system": { "type": [ "string", "array" ], "description": "System prompt: a string or text blocks." }, "tools": { "type": "array", "items": { "type": "object", "properties": { "name": { "type": "string", "description": "Tool name." }, "description": { "type": "string", "description": "What the tool does." }, "input_schema": { "type": "object", "description": "JSON Schema of the input." }, "type": { "type": "string", "description": "custom or absent; server tools (web search and others) are refused." } }, "required": [ "name" ] }, "description": "Client tools the model may call." }, "tool_choice": { "type": "object", "description": "{\"type\": \"auto\" | \"any\" | \"tool\" | \"none\", \"name\": ...}." }, "thinking": { "type": "object", "properties": { "type": { "type": "string", "description": "enabled or adaptive turns thinking on.", "enum": [ "enabled", "adaptive", "disabled" ] }, "budget_tokens": { "type": "integer", "description": "Accepted and ignored." } } }, "stream": { "type": "boolean", "description": "Stream the answer as server-sent events (see the 200 text/event-stream response). A client that reads nothing from the stream for 30 s is disconnected and its answer stops.", "default": false }, "prompt_cache_key": { "type": "string", "description": "Tags the request's session in the server log (as the X-Session-Id header does)." }, "metadata": { "type": "object", "description": "Accepted and ignored." }, "output_config": { "type": "object", "description": "effort is accepted and ignored. Not supported: format (structured output), answered with 400." }, "max_tokens": { "type": "integer", "description": "Most new tokens; default: up to the end of the context (--ctx).", "minimum": 1 }, "max_completion_tokens": { "type": "integer", "description": "Same as max_tokens (OpenAI's newer name); wins when both are set.", "minimum": 1 }, "temperature": { "type": "number", "description": "0 = greedy (default). Up to 4.", "minimum": 0, "maximum": 4, "default": 0.0 }, "top_p": { "type": "number", "description": "Nucleus sampling, in (0, 1].", "maximum": 1, "default": 1.0 }, "top_k": { "type": "integer", "description": "Keep the k most likely tokens; 0 = off.", "minimum": 0, "default": 0 }, "min_p": { "type": "number", "description": "Drop tokens below min_p times the top probability, in [0, 1).", "minimum": 0, "default": 0.0 }, "seed": { "type": "integer", "description": "Sampling seed; a random one when absent." }, "stop": { "type": [ "string", "array" ], "description": "A string or a list of non-empty strings; the text ends before the first one." }, "stop_sequences": { "type": "array", "description": "Same as stop (Anthropic's name), used when stop is absent.", "items": { "type": "string" } }, "repetition_penalty": { "type": "number", "description": "> 0; 1 = off.", "exclusiveMinimum": 0, "default": 1.0 }, "frequency_penalty": { "type": "number", "description": "-2 to 2; 0 = off.", "minimum": -2, "maximum": 2, "default": 0.0 }, "presence_penalty": { "type": "number", "description": "-2 to 2; 0 = off.", "minimum": -2, "maximum": 2, "default": 0.0 }, "n": { "type": "integer", "description": "Must be 1.", "enum": [ 1 ] } }, "required": [ "messages" ] }, "Message": { "type": "object", "properties": { "id": { "type": "string", "description": "msg_..." }, "type": { "type": "string", "description": "message", "const": "message" }, "role": { "type": "string", "description": "assistant" }, "model": { "type": "string", "description": "The model field of the request, else the model id." }, "content": { "type": "array", "items": { "$ref": "#/components/schemas/ContentBlock" } }, "stop_reason": { "type": [ "string", "null" ], "description": "end_turn, max_tokens, stop_sequence or tool_use; null in message_start.", "enum": [ "end_turn", "max_tokens", "stop_sequence", "tool_use", null ] }, "stop_sequence": { "type": [ "string", "null" ], "description": "The stop sequence that ended the text." }, "usage": { "$ref": "#/components/schemas/MessageUsage" } }, "required": [ "id", "type", "role", "model", "content", "stop_reason", "stop_sequence", "usage" ] }, "ContentBlock": { "type": "object", "properties": { "type": { "type": "string", "description": "text, thinking or tool_use.", "enum": [ "text", "thinking", "tool_use" ] }, "text": { "type": "string", "description": "text block." }, "thinking": { "type": "string", "description": "thinking block." }, "signature": { "type": "string", "description": "thinking block: mashbox-local." }, "id": { "type": "string", "description": "tool_use: toolu_..." }, "name": { "type": "string", "description": "tool_use: tool name." }, "input": { "type": "object", "description": "tool_use: arguments." } }, "required": [ "type" ] }, "MessageUsage": { "type": "object", "properties": { "input_tokens": { "type": "integer", "description": "Prompt tokens not served from the prefix cache." }, "cache_read_input_tokens": { "type": "integer", "description": "Prompt tokens served from the prefix cache." }, "cache_creation_input_tokens": { "type": "integer", "description": "Always 0." }, "output_tokens": { "type": "integer", "description": "New tokens." } } }, "MessageStreamEvent": { "type": "object", "properties": { "type": { "type": "string", "description": "Same as the SSE event name.", "enum": [ "message_start", "ping", "content_block_start", "content_block_delta", "content_block_stop", "message_delta", "message_stop", "error" ] }, "message": { "$ref": "#/components/schemas/Message" }, "index": { "type": "integer", "description": "Content block number." }, "content_block": { "$ref": "#/components/schemas/ContentBlock" }, "delta": { "type": "object", "description": "text_delta, thinking_delta, signature_delta, input_json_delta; on message_delta: stop_reason and stop_sequence." }, "usage": { "$ref": "#/components/schemas/MessageUsage" }, "error": { "type": "object", "description": "On error: type and message." } }, "required": [ "type" ], "description": "One `event:` + `data:` pair: message_start, ping, then per block content_block_start / _delta / _stop, then message_delta and message_stop." }, "CountTokensRequest": { "type": "object", "properties": { "model": { "type": "string", "description": "Accepted for compatibility; the server answers with the one model it runs." }, "messages": { "type": "array", "items": { "type": "object", "description": "As in MessagesRequest." } }, "system": { "type": [ "string", "array" ], "description": "As in MessagesRequest." }, "tools": { "type": "array", "items": { "type": "object", "description": "As in MessagesRequest." } }, "tool_choice": { "type": "object", "description": "As in MessagesRequest." }, "thinking": { "type": "object", "description": "As in MessagesRequest." } }, "required": [ "messages" ] }, "TokenCount": { "type": "object", "properties": { "input_tokens": { "type": "integer", "description": "Prompt tokens after the chat template." } }, "required": [ "input_tokens" ] }, "ResponsesRequest": { "type": "object", "properties": { "model": { "type": "string", "description": "Accepted for compatibility; the server answers with the one model it runs." }, "input": { "type": [ "string", "array" ], "description": "A string, or items: message (system, developer, user, assistant), function_call, function_call_output, custom_tool_call, custom_tool_call_output, local_shell_call, local_shell_call_output, reasoning. Text parts only: image, audio and file parts are refused with 400 (text-only model)." }, "instructions": { "type": "string", "description": "System prompt." }, "tools": { "type": "array", "items": { "type": "object", "description": "function, custom (grammar given to the model as text) or namespace tools. Hosted tools (web_search, file_search, code_interpreter, image_generation, computer_use_preview, mcp, local_shell) are dropped: the model does not see them." } }, "tool_choice": { "type": [ "string", "object" ], "description": "auto (default), none, required, or {\"type\": \"function\", \"function\": {\"name\": ...}}." }, "parallel_tool_calls": { "type": "boolean", "description": "Echoed in the response.", "default": false }, "reasoning": { "type": "object", "properties": { "effort": { "type": "string", "description": "Any value but none or minimal turns thinking on." } } }, "stream": { "type": "boolean", "description": "Stream the answer as server-sent events (see the 200 text/event-stream response). A client that reads nothing from the stream for 30 s is disconnected and its answer stops.", "default": false }, "prompt_cache_key": { "type": "string", "description": "Tags the request's session in the server log (as the X-Session-Id header does)." }, "store": { "type": "boolean", "description": "Accepted and ignored." }, "client_metadata": { "type": "object", "description": "Accepted and ignored." }, "include": { "type": "array", "description": "Accepted and ignored, except an entry naming logprobs: 400 (log probabilities are not supported)." }, "top_logprobs": { "type": "integer", "description": "Not supported (log probabilities): a request that sets it gets 400 naming the field." }, "text": { "type": "object", "description": "Only format {\"type\": \"text\"}. Not supported: other formats (structured output), answered with 400." }, "max_output_tokens": { "type": "integer", "description": "Most new tokens; default: up to the end of the context (--ctx).", "minimum": 1 }, "max_tokens": { "type": "integer", "description": "Most new tokens (>= 1).", "minimum": 1 }, "max_completion_tokens": { "type": "integer", "description": "Same as max_tokens (OpenAI's newer name); wins when both are set.", "minimum": 1 }, "temperature": { "type": "number", "description": "0 = greedy (default). Up to 4.", "minimum": 0, "maximum": 4, "default": 0.0 }, "top_p": { "type": "number", "description": "Nucleus sampling, in (0, 1].", "maximum": 1, "default": 1.0 }, "top_k": { "type": "integer", "description": "Keep the k most likely tokens; 0 = off.", "minimum": 0, "default": 0 }, "min_p": { "type": "number", "description": "Drop tokens below min_p times the top probability, in [0, 1).", "minimum": 0, "default": 0.0 }, "seed": { "type": "integer", "description": "Sampling seed; a random one when absent." }, "stop": { "type": [ "string", "array" ], "description": "A string or a list of non-empty strings; the text ends before the first one." }, "stop_sequences": { "type": "array", "description": "Same as stop (Anthropic's name), used when stop is absent.", "items": { "type": "string" } }, "repetition_penalty": { "type": "number", "description": "> 0; 1 = off.", "exclusiveMinimum": 0, "default": 1.0 }, "frequency_penalty": { "type": "number", "description": "-2 to 2; 0 = off.", "minimum": -2, "maximum": 2, "default": 0.0 }, "presence_penalty": { "type": "number", "description": "-2 to 2; 0 = off.", "minimum": -2, "maximum": 2, "default": 0.0 }, "n": { "type": "integer", "description": "Must be 1.", "enum": [ 1 ] } }, "required": [ "input" ] }, "Response": { "type": "object", "properties": { "id": { "type": "string", "description": "resp_..." }, "object": { "type": "string", "description": "response", "const": "response" }, "created_at": { "type": "integer", "description": "Unix time." }, "status": { "type": "string", "description": "completed, or incomplete when max_output_tokens ran out.", "enum": [ "in_progress", "completed", "incomplete" ] }, "model": { "type": "string", "description": "The model field of the request, else the model id." }, "output": { "type": "array", "items": { "$ref": "#/components/schemas/ResponseItem" } }, "parallel_tool_calls": { "type": "boolean", "description": "From the request." }, "tool_choice": { "type": [ "string", "object" ], "description": "From the request." }, "tools": { "type": "array", "description": "From the request." }, "usage": { "type": [ "object", "null" ], "description": "input_tokens, input_tokens_details.cached_tokens, output_tokens, output_tokens_details.reasoning_tokens, total_tokens; null while in progress." }, "incomplete_details": { "type": "object", "properties": { "reason": { "type": "string", "description": "max_output_tokens" } } } }, "required": [ "id", "object", "created_at", "status", "model", "output", "usage" ] }, "ResponseItem": { "type": "object", "properties": { "type": { "type": "string", "description": "message, reasoning, function_call or custom_tool_call.", "enum": [ "message", "reasoning", "function_call", "custom_tool_call" ] }, "id": { "type": "string", "description": "Item id." }, "status": { "type": "string", "description": "in_progress or completed." }, "role": { "type": "string", "description": "message: assistant." }, "content": { "type": "array", "description": "message: output_text parts; reasoning: reasoning_text parts." }, "summary": { "type": "array", "description": "reasoning: empty." }, "call_id": { "type": "string", "description": "Call id to answer with an *_output item." }, "name": { "type": "string", "description": "Tool name." }, "arguments": { "type": "string", "description": "function_call: arguments, JSON." }, "input": { "type": "string", "description": "custom_tool_call: raw input." }, "namespace": { "type": "string", "description": "function_call: the namespace tool it came from." } }, "required": [ "type", "id" ] }, "ResponseStreamEvent": { "type": "object", "properties": { "type": { "type": "string", "description": "Same as the SSE event name.", "enum": [ "response.created", "response.in_progress", "response.output_item.added", "response.output_item.done", "response.content_part.added", "response.content_part.done", "response.output_text.delta", "response.output_text.done", "response.reasoning_text.delta", "response.reasoning_text.done", "response.function_call_arguments.delta", "response.function_call_arguments.done", "response.custom_tool_call_input.delta", "response.custom_tool_call_input.done", "response.completed", "response.incomplete", "error", "response.failed" ] }, "sequence_number": { "type": "integer", "description": "1, 2, 3, ..." }, "response": { "$ref": "#/components/schemas/Response" }, "item": { "$ref": "#/components/schemas/ResponseItem" }, "item_id": { "type": "string", "description": "Item the event belongs to." }, "output_index": { "type": "integer", "description": "Item number." }, "content_index": { "type": "integer", "description": "Part number." }, "delta": { "type": "string", "description": "New text or arguments." }, "text": { "type": "string", "description": "Whole text (done events)." }, "part": { "type": "object", "description": "Content part." }, "arguments": { "type": "string", "description": "Whole arguments (done event)." }, "input": { "type": "string", "description": "Whole input (done event)." } }, "required": [ "type", "sequence_number" ], "description": "One `event:` + `data:` pair. An error after the stream started is sent as event `error`, then `response.failed` (the response with status failed and its error)." } } } }