feat(js/plugins/mcp): support for multipart tools

js/ai/src/index.ts

@@ -157,6 +157,7 @@ export {
   respondTool,
   restartTool,
   type InterruptConfig,
+  type MultipartToolAction,
   type ToolAction,
   type ToolArgument,
   type ToolConfig,

js/genkit/src/common.ts

@@ -91,6 +91,7 @@ export {
   type ModelReference,
   type ModelRequest,
   type ModelResponseData,
+  type MultipartToolAction,
   type OutputOptions,
   type Part,
   type PromptAction,

js/genkit/src/tool.ts

@@ -20,6 +20,7 @@ export {
   interrupt,
   toToolDefinition,
   tool,
+  type MultipartToolAction,
   type ToolAction,
   type ToolArgument,
   type ToolConfig,

js/plugins/google-genai/src/common/converters.ts

@@ -162,21 +162,55 @@ function toGeminiToolRequest(part: Part): GeminiPart {
 }
 
 function toGeminiToolResponse(part: Part): GeminiPart {
-  if (!part.toolResponse?.output) {
-    throw Error('Invalid ToolResponsePart: output was missing.');
+  if (part.toolResponse?.output === undefined && !part.toolResponse?.content) {
+    throw Error(
+      'Invalid ToolResponsePart: output or content must be provided.'
+    );
   }
+
+  let responseContent: any = part.toolResponse?.output;
+
+  const functionResponseParts: GeminiPart[] = [];
+  if (part.toolResponse?.content) {
+    const texts: string[] = [];
+    for (const p of part.toolResponse.content) {
+      if (typeof p.text === 'string') {
+        texts.push(p.text);
+      } else if (p.media) {
+        functionResponseParts.push(toGeminiPart(p));
+      }
+    }
+
+    if (texts.length > 0) {
+      if (responseContent === undefined) {
+        responseContent = { text: texts.join('\n') };
+      } else if (
+        typeof responseContent === 'object' &&
+        responseContent !== null
+      ) {
+        responseContent = { ...responseContent, text: texts.join('\n') };
+      }
+    }
+  }
+
+  if (responseContent === undefined) {
+    responseContent = {};
+  }
+
   const functionResponse: GeminiPart['functionResponse'] = {
-    name: part.toolResponse.name,
+    name: part.toolResponse!.name,
     response: {
-      name: part.toolResponse.name,
-      content: part.toolResponse.output,
+      name: part.toolResponse!.name,
+      content: responseContent,
     },
   };
-  if (part.toolResponse.content) {
-    functionResponse.parts = part.toolResponse.content.map(toGeminiPart);
+
+  if (functionResponseParts.length > 0) {
+    functionResponse.parts = functionResponseParts;
   }
-  if (part.toolResponse.ref) {
-    functionResponse.id = part.toolResponse.ref;
+
+  if (part.toolResponse!.ref) {
+    functionResponse.id = part.toolResponse!.ref;
   }
   return maybeAddGeminiThoughtSignatureAndMetadata(part, {
     functionResponse,

js/plugins/google-genai/tests/common/converters_test.ts

@@ -165,6 +165,85 @@ describe('toGeminiMessage', () => {
         ],
       },
     },
+    {
+      should:
+        'should transform genkit message (tool response with missing output and mixed content) correctly',
+      inputMessage: {
+        role: 'tool',
+        content: [
+          {
+            toolResponse: {
+              name: 'screenshot',
+              ref: '1',
+              content: [
+                { text: 'this is a test' },
+                {
+                  media: {
+                    contentType: 'image/png',
+                    url: 'data:image/png;base64,SHORTENED_BASE64_DATA',
+                  },
+                },
+              ],
+            },
+          },
+        ],
+      },
+      expectedOutput: {
+        role: 'function',
+        parts: [
+          {
+            functionResponse: {
+              id: '1',
+              name: 'screenshot',
+              response: {
+                name: 'screenshot',
+                content: { text: 'this is a test' },
+              },
+              parts: [
+                {
+                  inlineData: {
+                    mimeType: 'image/png',
+                    data: 'SHORTENED_BASE64_DATA',
+                  },
+                },
+              ],
+            },
+          },
+        ],
+      },
+    },
+    {
+      should:
+        'should transform genkit message (tool response with output object and text content) correctly',
+      inputMessage: {
+        role: 'tool',
+        content: [
+          {
+            toolResponse: {
+              name: 'screenshot',
+              ref: '2',
+              output: { status: 'ok' },
+              content: [{ text: 'this is a test' }],
+            },
+          },
+        ],
+      },
+      expectedOutput: {
+        role: 'function',
+        parts: [
+          {
+            functionResponse: {
+              id: '2',
+              name: 'screenshot',
+              response: {
+                name: 'screenshot',
+                content: { status: 'ok', text: 'this is a test' },
+              },
+            },
+          },
+        ],
+      },
+    },
     {
       should:
         'should transform genkit message (inline base64 image content) correctly',

js/plugins/mcp/package.json

@@ -39,7 +39,7 @@
   "license": "Apache-2.0",
   "peerDependencies": {
     "genkit": "workspace:^",
-    "@modelcontextprotocol/sdk": "^1.13.0"
+    "@modelcontextprotocol/sdk": "^1.29.0"
   },
   "devDependencies": {
     "get-port": "^5.1.0",

js/plugins/mcp/src/client/client.ts

@@ -28,6 +28,7 @@ import {
   type DynamicResourceAction,
   type ExecutablePrompt,
   type Genkit,
+  type MultipartToolAction,
   type PromptGenerateOptions,
   type ToolAction,
 } from 'genkit';
@@ -91,7 +92,7 @@ export type McpServerConfig = (
  * Configuration options for an individual `GenkitMcpClient` instance.
  * This defines how the client connects to a single MCP server and how it behaves.
  */
-export type McpClientOptions = {
+export type McpClientOptions<M extends boolean = false> = {
   /** Client name to advertise to the server. */
   name: string;
   /** Name for the server, defaults to the server's advertised name. */
@@ -108,15 +109,18 @@ export type McpClientOptions = {
    * simplified for better compatibility with Genkit's typical data structures.
    */
   rawToolResponses?: boolean;
+  /** If true, tools will be registered as multipart tool.v2 actions. */
+  multipart?: M;
   /** The server configuration to connect. */
   mcpServer: McpServerConfig;
   /** Manually supply a session id for HTTP streaming clients if desired. */
   sessionId?: string;
 };
 
-export type McpClientOptionsWithCache = McpClientOptions & {
-  cacheTtlMillis?: number;
-};
+export type McpClientOptionsWithCache<M extends boolean = false> =
+  McpClientOptions<M> & {
+    cacheTtlMillis?: number;
+  };
 
 /**
  * Represents a client connection to a single MCP (Model Context Protocol) server.
@@ -126,7 +130,7 @@ export type McpClientOptionsWithCache = McpClientOptions & {
  * An instance of `GenkitMcpClient` is typically managed by a `GenkitMcpHost`
  * when dealing with multiple MCP server connections.
  */
-export class GenkitMcpClient {
+export class GenkitMcpClient<Multipart extends boolean = false> {
   _server?: McpServerRef;
   private _dynamicActionProvider: DynamicActionProviderAction | undefined;
 
@@ -136,6 +140,7 @@ export class GenkitMcpClient {
   private version: string;
   private serverConfig: McpServerConfig;
   private rawToolResponses?: boolean;
+  private multipart?: boolean;
   private disabled: boolean;
   private roots?: Root[];
 
@@ -145,12 +150,13 @@ export class GenkitMcpClient {
   }[] = [];
   private _ready = false;
 
-  constructor(options: McpClientOptions) {
+  constructor(options: McpClientOptions<Multipart>) {
     this.name = options.name;
     this.suppliedServerName = options.serverName;
     this.version = options.version || '1.0.0';
     this.serverConfig = options.mcpServer;
     this.rawToolResponses = !!options.rawToolResponses;
+    this.multipart = !!options.multipart;
     this.disabled = !!options.mcpServer.disabled;
     this.roots = options.mcpServer.roots;
     this.sessionId = options.sessionId;
@@ -341,23 +347,41 @@ export class GenkitMcpClient {
    * Fetches all tools available through this client, if the server
    * configuration is not disabled.
    */
-  async getActiveTools(ai: Genkit): Promise<ToolAction[]> {
+  async getActiveTools(
+    ai: Genkit
+  ): Promise<(Multipart extends true ? MultipartToolAction : ToolAction)[]> {
     await this.ready();
-    let tools: ToolAction[] = [];
 
     if (this._server) {
       const capabilities = this._server.client.getServerCapabilities();
-      if (capabilities?.tools)
-        tools.push(
-          ...(await fetchDynamicTools(ai, this._server.client, {
+      if (capabilities?.tools) {
+        if (this.multipart) {
+          const tools = await fetchDynamicTools(ai, this._server.client, {
             rawToolResponses: this.rawToolResponses,
+            multipart: true,
             serverName: this.serverName,
             name: this.name,
-          }))
-        );
+          });
+          return tools as unknown as (Multipart extends true
+            ? MultipartToolAction
+            : ToolAction)[];
+        } else {
+          const tools = await fetchDynamicTools(ai, this._server.client, {
+            rawToolResponses: this.rawToolResponses,
+            multipart: false,
+            serverName: this.serverName,
+            name: this.name,
+          });
+          return tools as unknown as (Multipart extends true
+            ? MultipartToolAction
+            : ToolAction)[];
+        }
+      }
     }
 
-    return tools;
+    return [] as unknown as (Multipart extends true
+      ? MultipartToolAction
+      : ToolAction)[];
   }
 
   /**