Skip to content

feat: populate ToolCallingRecord.images when tool returns ToolResult #3981

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
KIA-Er wants to merge 2 commits into
base: master
Choose a base branch
from
+79 −55
Open

feat: populate ToolCallingRecord.images when tool returns ToolResult #3981

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8f350f5
feat: populate ToolCallingRecord.images when tool returns ToolResult
KIA-Er Apr 1, 2026
a587678
Update chat_agent.py
fengju0213 Apr 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 79 additions & 55 deletions camel/agents/chat_agent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4211,62 +4211,11 @@ def _record_tool_calling(
cast(List[MemoryRecord], func_records),
)

# Extract and inject images if result is ToolResult
images = None
if isinstance(result, ToolResult) and result.images:
try:
import base64
import io

try:
from PIL import Image
except ImportError:
logger.warning(
f"Tool '{func_name}' returned images but PIL "
"is not installed. Install with: pip install "
"Pillow. Skipping visual context injection."
)
# Continue without injecting images
result = (
result.text if hasattr(result, 'text') else str(result)
)
else:
logger.info(
f"Tool '{func_name}' returned ToolResult with "
f"{len(result.images)} image(s), injecting into "
"context"
)

# Convert base64 images to PIL Image objects
pil_images: List[Union[Image.Image, str]] = []
for img_data in result.images:
if img_data.startswith('data:image/'):
# Extract base64 data
base64_str = img_data.split(',', 1)[1]
img_bytes = base64.b64decode(base64_str)
pil_img = Image.open(io.BytesIO(img_bytes))
pil_images.append(pil_img)

if pil_images:
# Create a user message with the image(s)
visual_msg = BaseMessage.make_user_message(
role_name="Tool",
content=f"[Visual output from {func_name}]",
image_list=pil_images,
)

# Inject into conversation context
self.update_memory(
visual_msg,
OpenAIBackendRole.USER,
return_records=False,
)
logger.info(
f"Successfully injected {len(pil_images)} "
"image(s) into agent context"
)
except Exception as e:
logger.error(
f"Failed to inject visual content from {func_name}: {e}"
)
images = result.images
self._inject_tool_images_into_memory(func_name, result.images)

# Record information about this tool call
# Note: tool_record contains the original result for the caller,
Expand All @@ -4276,10 +4225,67 @@ def _record_tool_calling(
args=args,
result=result,
tool_call_id=tool_call_id,
images=images,
)
self._update_last_tool_call_state(tool_record)
return tool_record

def _inject_tool_images_into_memory(
self, func_name: str, image_list: List[str]
) -> None:
r"""Inject images from a tool result into agent memory.

Args:
func_name (str): The name of the tool that returned images.
image_list (List[str]): List of base64-encoded image strings.
"""
try:
import base64
import io

try:
from PIL import Image
except ImportError:
logger.warning(
f"Tool '{func_name}' returned images but PIL "
"is not installed. Install with: pip install "
"Pillow. Skipping visual context injection."
)
return

logger.info(
f"Tool '{func_name}' returned ToolResult with "
f"{len(image_list)} image(s), injecting into context"
)

pil_images: List[Union[Image.Image, str]] = []
for img_data in image_list:
if img_data.startswith('data:image/'):
base64_str = img_data.split(',', 1)[1]
img_bytes = base64.b64decode(base64_str)
pil_img = Image.open(io.BytesIO(img_bytes))
pil_images.append(pil_img)

if pil_images:
visual_msg = BaseMessage.make_user_message(
role_name="Tool",
content=f"[Visual output from {func_name}]",
image_list=pil_images,
)
self.update_memory(
visual_msg,
OpenAIBackendRole.USER,
return_records=False,
)
logger.info(
f"Successfully injected {len(pil_images)} "
"image(s) into agent context"
)
except Exception as e:
logger.error(
f"Failed to inject visual content from {func_name}: {e}"
)

def _stream(
self,
input_message: Union[BaseMessage, str],
Expand Down Expand Up @@ -5068,11 +5074,20 @@ def _execute_tool_from_stream_data(
# Record only the tool result message
self.update_memory(func_msg, OpenAIBackendRole.FUNCTION)

# Extract and inject images if result is ToolResult
images = None
if isinstance(result, ToolResult) and result.images:
images = result.images
self._inject_tool_images_into_memory(
function_name, result.images
)

tool_record = ToolCallingRecord(
tool_name=function_name,
args=args,
result=result,
tool_call_id=tool_call_id,
images=images,
)
self._update_last_tool_call_state(tool_record)
return tool_record
Expand Down Expand Up @@ -5224,11 +5239,20 @@ async def _aexecute_tool_from_stream_data(
)
self.update_memory(func_msg, OpenAIBackendRole.FUNCTION)

# Extract and inject images if result is ToolResult
images = None
if isinstance(result, ToolResult) and result.images:
images = result.images
self._inject_tool_images_into_memory(
function_name, result.images
)

tool_record = ToolCallingRecord(
tool_name=function_name,
args=args,
result=result,
tool_call_id=tool_call_id,
images=images,
)
self._update_last_tool_call_state(tool_record)
return tool_record
Expand Down
Loading