feat: add QR login and browser-backed web tools
This commit is contained in:
Azimkin
228
app/llm.py
Normal file
228
app/llm.py
Normal file
@@ -0,0 +1,228 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from typing import Any, cast
|
||||
|
||||
from micro_agents import (
|
||||
Conversation,
|
||||
LlmMessage,
|
||||
Media,
|
||||
Tool,
|
||||
ToolCallMessage,
|
||||
ToolCallResponseMessage,
|
||||
UserMessage,
|
||||
)
|
||||
from micro_agents.conversation import Message, ReasoningMessage
|
||||
from micro_agents.llm import OpenAIBasedLLM
|
||||
|
||||
|
||||
class ChatCompletionsLLM(OpenAIBasedLLM):
|
||||
"""micro-agents adapter for OpenAI-compatible Chat Completions APIs."""
|
||||
|
||||
async def ask(
|
||||
self,
|
||||
conversation: Conversation,
|
||||
tools: list[Tool],
|
||||
*,
|
||||
instructions: str | None = None,
|
||||
thinking: bool | None = None,
|
||||
) -> list[Message]:
|
||||
messages = self._serialize_conversation(conversation, instructions)
|
||||
serialized_tools = self._serialize_tools(tools)
|
||||
request: dict[str, Any] = {
|
||||
"model": self._model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"extra_body": {
|
||||
"thinking": {"type": "enabled" if thinking else "disabled"},
|
||||
},
|
||||
}
|
||||
if serialized_tools:
|
||||
request["tools"] = serialized_tools
|
||||
|
||||
response = await self._ai.chat.completions.create(**cast(Any, request))
|
||||
if not response.choices:
|
||||
raise RuntimeError("Chat Completions API returned no choices")
|
||||
|
||||
message = response.choices[0].message
|
||||
result: list[Message] = []
|
||||
reasoning = getattr(message, "reasoning_content", None)
|
||||
if reasoning:
|
||||
result.append(ReasoningMessage(str(reasoning)))
|
||||
if message.content:
|
||||
result.append(LlmMessage(message.content))
|
||||
|
||||
for call in message.tool_calls or ():
|
||||
try:
|
||||
arguments = json.loads(call.function.arguments)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError("Model returned invalid tool-call arguments") from exc
|
||||
if not isinstance(arguments, dict):
|
||||
raise ValueError("Model tool-call arguments must be an object")
|
||||
result.append(ToolCallMessage(call.id, call.function.name, arguments))
|
||||
|
||||
return result
|
||||
|
||||
def _serialize_tools(self, tools: list[Tool]) -> list[dict[str, object]]:
|
||||
response_tools = super()._serialize_tools(tools)
|
||||
result: list[dict[str, object]] = []
|
||||
for tool in response_tools:
|
||||
function = {
|
||||
key: tool[key]
|
||||
for key in ("name", "description", "parameters")
|
||||
if key in tool and tool[key] is not None
|
||||
}
|
||||
result.append({"type": "function", "function": function})
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def _serialize_conversation(
|
||||
cls,
|
||||
conversation: Conversation,
|
||||
instructions: str | None,
|
||||
) -> list[dict[str, object]]:
|
||||
result: list[dict[str, object]] = []
|
||||
if instructions:
|
||||
result.append({"role": "system", "content": instructions})
|
||||
|
||||
pending_assistant: dict[str, object] | None = None
|
||||
pending_attachments: list[Media] = []
|
||||
|
||||
def flush_assistant() -> None:
|
||||
nonlocal pending_assistant
|
||||
if pending_assistant is None:
|
||||
return
|
||||
pending_assistant.setdefault("content", "")
|
||||
result.append(pending_assistant)
|
||||
pending_assistant = None
|
||||
|
||||
def flush_attachments() -> None:
|
||||
if not pending_attachments:
|
||||
return
|
||||
result.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": cls._serialize_user_content(
|
||||
"Attachments returned by tools:",
|
||||
pending_attachments,
|
||||
),
|
||||
}
|
||||
)
|
||||
pending_attachments.clear()
|
||||
|
||||
for message in conversation.messages:
|
||||
if isinstance(message, ToolCallResponseMessage):
|
||||
flush_assistant()
|
||||
result.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": message.unique_id(),
|
||||
"content": message.text(),
|
||||
}
|
||||
)
|
||||
pending_attachments.extend(message.attachments())
|
||||
continue
|
||||
|
||||
flush_attachments()
|
||||
if isinstance(message, UserMessage):
|
||||
flush_assistant()
|
||||
result.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": cls._serialize_user_content(
|
||||
message.message(),
|
||||
message.media(),
|
||||
),
|
||||
}
|
||||
)
|
||||
elif isinstance(message, ReasoningMessage):
|
||||
flush_assistant()
|
||||
pending_assistant = {
|
||||
"role": "assistant",
|
||||
"reasoning_content": message.text(),
|
||||
}
|
||||
elif isinstance(message, LlmMessage):
|
||||
if pending_assistant is None:
|
||||
pending_assistant = {"role": "assistant"}
|
||||
pending_assistant["content"] = message.message()
|
||||
elif isinstance(message, ToolCallMessage):
|
||||
if pending_assistant is None:
|
||||
pending_assistant = {"role": "assistant"}
|
||||
calls = cast(list[dict[str, object]], pending_assistant.setdefault("tool_calls", []))
|
||||
calls.append(
|
||||
{
|
||||
"id": message.unique_id(),
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": message.tool_name(),
|
||||
"arguments": json.dumps(
|
||||
message.arguments(),
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
),
|
||||
},
|
||||
}
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown conversation message: {type(message).__name__}")
|
||||
|
||||
flush_assistant()
|
||||
flush_attachments()
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def _serialize_user_content(
|
||||
cls,
|
||||
text: str,
|
||||
media_items: list[Media],
|
||||
) -> str | list[dict[str, object]]:
|
||||
if not media_items:
|
||||
return text
|
||||
content: list[dict[str, object]] = []
|
||||
if text:
|
||||
content.append({"type": "text", "text": text})
|
||||
content.extend(cls._serialize_media(media) for media in media_items)
|
||||
return content
|
||||
|
||||
@staticmethod
|
||||
def _serialize_media(media: Media) -> dict[str, object]:
|
||||
if media.kind == "image":
|
||||
if media.url is not None:
|
||||
url = media.url
|
||||
elif media.data is not None:
|
||||
mime_type = media.mime_type or "image/jpeg"
|
||||
encoded = base64.b64encode(media.data).decode("ascii")
|
||||
url = f"data:{mime_type};base64,{encoded}"
|
||||
else:
|
||||
raise ValueError("Chat Completions image input requires data or URL")
|
||||
return {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": url, "detail": media.detail},
|
||||
}
|
||||
|
||||
if media.kind == "audio":
|
||||
if media.data is None:
|
||||
raise ValueError("Chat Completions audio input requires inline data")
|
||||
mime_type = media.mime_type or ""
|
||||
audio_format = "mp3" if mime_type == "audio/mpeg" else "wav"
|
||||
return {
|
||||
"type": "input_audio",
|
||||
"input_audio": {
|
||||
"data": base64.b64encode(media.data).decode("ascii"),
|
||||
"format": audio_format,
|
||||
},
|
||||
}
|
||||
|
||||
file: dict[str, object] = {}
|
||||
if media.file_id is not None:
|
||||
file["file_id"] = media.file_id
|
||||
elif media.data is not None:
|
||||
mime_type = media.mime_type or "application/octet-stream"
|
||||
encoded = base64.b64encode(media.data).decode("ascii")
|
||||
file["file_data"] = f"data:{mime_type};base64,{encoded}"
|
||||
file["filename"] = media.filename or "file"
|
||||
elif media.url is not None:
|
||||
file["file_data"] = media.url
|
||||
file["filename"] = media.filename or "file"
|
||||
return {"type": "file", "file": file}
|
||||
Reference in New Issue
Block a user