Support logprobs in Responses API

[irar2]

[mayabar]

Request/response structure

completions

request

• logprobs: int | null

response

• choices
  - ...
  - logprobs
  - tokens: array of strings
  - token_logprobs: array of numbers
  - top_logprobs: map string->int
  - text_offset: array of ints

chat completions

request

• logprobs: bool
• top_logprobs: int | null

response

• choices
  - ...
  - logprobs
  - content: array of objects
  - token
  - logprob
  - bytes: array of bytes
  - top_logprobs: attay of logprob objects

responses

request

• include: array of strings, should contain "message.output_text.logprobs"
• top_logprobs: int | null

response

• output
  • content
    • logprobs: array of objects
      • token: string
      • bytes: array of bytes
      • logprob: float
      • top_logprobs: empty array if top_logprobs is not defined in the reqiest, array of logprobs objects if top_logprobs is defined

[mayabar]

Examples:

curl -s http://localhost:8000/v1/responses \
>  -H 'Content-Type: application/json' \
>  -d '{"model":"Qwen/Qwen2-0.5B","input":"hi", "max_output_tokens":4,"include":["message.output_text.logprobs"], "ignore_eos": true}' | jq

Response

``` { "id": "resp_a21b34d4649fe523", "created_at": 1777533961, "incomplete_details": {   "reason": "max_output_tokens" }, "instructions": null, "metadata": null, "model": "Qwen/Qwen2-0.5B", "object": "response", "output": [   {    "id": "msg_bd9316d779c55ef7",    "content": [     {      "annotations": [],      "text": "根据.annotation.cell().",      "type": "output_text",      "logprobs": [       {        "token": "根据",        "bytes": [         230,         160,         185,         230,         141,         174        ],        "logprob": -3.7662811279296875,        "top_logprobs": []       },       {        "token": ".annotation",        "bytes": [         46,         97,         110,         110,         111,         116,         97,         116,         105,         111,         110        ],        "logprob": -11.680919647216797,        "top_logprobs": []       },       {        "token": ".cell",        "bytes": [         46,         99,         101,         108,         108        ],        "logprob": -10.4017972946167,        "top_logprobs": []       },       {        "token": "().",        "bytes": [         40,         41,         46        ],        "logprob": -4.900529861450195,        "top_logprobs": []       }      ]     }    ],    "role": "assistant",    "status": "completed",    "type": "message"   } ], "parallel_tool_calls": true, "temperature": 1.0, "tool_choice": "auto", "tools": [], "top_p": 1.0, "background": false, "max_output_tokens": 4, "max_tool_calls": null, "previous_response_id": null, "prompt": null, "reasoning": null, "service_tier": "auto", "status": "incomplete", "text": null, "top_logprobs": 0, "truncation": "disabled", "usage": {   "input_tokens": 19,   "input_tokens_details": {    "cached_tokens": 16,    "input_tokens_per_turn": [],    "cached_tokens_per_turn": []   },   "output_tokens": 4,   "output_tokens_details": {    "reasoning_tokens": 0,    "tool_output_tokens": 0,    "output_tokens_per_turn": [],    "tool_output_tokens_per_turn": []   },   "total_tokens": 23 }, "user": null, "input_messages": null, "output_messages": null } ```

curl -s http://localhost:8000/v1/responses \
>  -H 'Content-Type: application/json' \
>  -d '{"model":"Qwen/Qwen2-0.5B","input":"hi","top_logprobs":3,"max_output_tokens":4,"include":["message.output_text.logprobs"], "ignore_eos": true}' | jq

Response

``` { "id": "resp_9618be13a8878b58", "created_at": 1777533905, "incomplete_details": {   "reason": "max_output_tokens" }, "instructions": null, "metadata": null, "model": "Qwen/Qwen2-0.5B", "object": "response", "output": [   {    "id": "msg_904984a9efe29887",    "content": [     {      "annotations": [],      "text": ".txt files present to",      "type": "output_text",      "logprobs": [       {        "token": ".txt",        "bytes": [         46,         116,         120,         116        ],        "logprob": -9.672531127929688,        "top_logprobs": [         {          "token": ".txt",          "bytes": [           46,           116,           120,           116          ],          "logprob": -9.672531127929688         },         {          "token": "The",          "bytes": [           84,           104,           101          ],          "logprob": -2.4537811279296875         },         {          "token": "Based",          "bytes": [           66,           97,           115,           101,           100          ],          "logprob": -2.7037811279296875         }        ]       },       {        "token": " files",        "bytes": [         32,         102,         105,         108,         101,         115        ],        "logprob": -3.707690477371216,        "top_logprobs": [         {          "token": " files",          "bytes": [           32,           102,           105,           108,           101,           115          ],          "logprob": -3.707690477371216         },         {          "token": " file",          "bytes": [           32,           102,           105,           108,           101          ],          "logprob": -1.4576904773712158         },         {          "token": "\n",          "bytes": [           10          ],          "logprob": -2.145190477371216         }        ]       },       {        "token": " present",        "bytes": [         32,         112,         114,         101,         115,         101,         110,         116        ],        "logprob": -7.353982925415039,        "top_logprobs": [         {          "token": " present",          "bytes": [           32,           112,           114,           101,           115,           101,           110,           116          ],          "logprob": -7.353982925415039         },         {          "token": " in",          "bytes": [           32,           105,           110          ],          "logprob": -2.291482925415039         },         {          "token": ".",          "bytes": [           46          ],          "logprob": -2.291482925415039         }        ]       },       {        "token": " to",        "bytes": [         32,         116,         111        ],        "logprob": -5.673874855041504,        "top_logprobs": [         {          "token": " to",          "bytes": [           32,           116,           111          ],          "logprob": -5.673874855041504         },         {          "token": " in",          "bytes": [           32,           105,           110          ],          "logprob": -0.6738747358322144         },         {          "token": " on",          "bytes": [           32,           111,           110          ],          "logprob": -2.611374855041504         }        ]       }      ]     }    ],    "role": "assistant",    "status": "completed",    "type": "message"   } ], "parallel_tool_calls": true, "temperature": 1.0, "tool_choice": "auto", "tools": [], "top_p": 1.0, "background": false, "max_output_tokens": 4, "max_tool_calls": null, "previous_response_id": null, "prompt": null, "reasoning": null, "service_tier": "auto", "status": "incomplete", "text": null, "top_logprobs": 3, "truncation": "disabled", "usage": {   "input_tokens": 19,   "input_tokens_details": {    "cached_tokens": 16,    "input_tokens_per_turn": [],    "cached_tokens_per_turn": []   },   "output_tokens": 4,   "output_tokens_details": {    "reasoning_tokens": 0,    "tool_output_tokens": 0,    "output_tokens_per_turn": [],    "tool_output_tokens_per_turn": []   },   "total_tokens": 23 }, "user": null, "input_messages": null, "output_messages": null } ```