From 80138a684f9923d7d2aef56fa205f8e8797d0eb3 Mon Sep 17 00:00:00 2001 From: Tom C Date: Thu, 18 Dec 2025 13:15:01 +1000 Subject: [PATCH 1/3] feat(google): preserve thought signatures for Gemini 3+ function calling - Add thoughtSignature field to FunctionCall class in chat_context.ts - Capture thoughtSignature from API response in google plugin llm.ts - Include thoughtSignature when converting back to Google format - Preserve thoughtSignature in voice/generation.ts when recreating FunctionCall - Update @google/genai to v1.34.0 for thoughtSignature support --- agents/src/llm/chat_context.ts | 30 ++++- agents/src/llm/provider_format/google.ts | 8 +- agents/src/voice/generation.ts | 2 + plugins/google/package.json | 2 +- plugins/google/src/llm.ts | 2 + pnpm-lock.yaml | 133 ++++++++++++++--------- 6 files changed, 120 insertions(+), 57 deletions(-) diff --git a/agents/src/llm/chat_context.ts b/agents/src/llm/chat_context.ts index cddd08d62..3a7ae179b 100644 --- a/agents/src/llm/chat_context.ts +++ b/agents/src/llm/chat_context.ts @@ -189,19 +189,35 @@ export class FunctionCall { createdAt: number; + /** + * Opaque signature for Gemini thinking mode. + * When using Gemini 3+ models with thinking enabled, this signature must be + * preserved and returned with function responses to maintain thought context. + */ + thoughtSignature?: string; + constructor(params: { callId: string; name: string; args: string; id?: string; createdAt?: number; + thoughtSignature?: string; }) { - const { callId, name, args, id = shortuuid('item_'), createdAt = Date.now() } = params; + const { + callId, + name, + args, + id = shortuuid('item_'), + createdAt = Date.now(), + thoughtSignature, + } = params; this.id = id; this.callId = callId; this.args = args; this.name = name; this.createdAt = createdAt; + this.thoughtSignature = thoughtSignature; } static create(params: { @@ -210,6 +226,7 @@ export class FunctionCall { args: string; id?: string; createdAt?: number; + thoughtSignature?: string; }) { return new FunctionCall(params); } @@ -224,6 +241,10 @@ export class FunctionCall { args: this.args, }; + if (this.thoughtSignature) { + result.thoughtSignature = this.thoughtSignature; + } + if (!excludeTimestamp) { result.createdAt = this.createdAt; } @@ -602,7 +623,12 @@ export class ChatContext { return false; } } else if (a.type === 'function_call' && b.type === 'function_call') { - if (a.name !== b.name || a.callId !== b.callId || a.args !== b.args) { + if ( + a.name !== b.name || + a.callId !== b.callId || + a.args !== b.args || + a.thoughtSignature !== b.thoughtSignature + ) { return false; } } else if (a.type === 'function_call_output' && b.type === 'function_call_output') { diff --git a/agents/src/llm/provider_format/google.ts b/agents/src/llm/provider_format/google.ts index abe595c39..45c68295d 100644 --- a/agents/src/llm/provider_format/google.ts +++ b/agents/src/llm/provider_format/google.ts @@ -67,13 +67,17 @@ export async function toChatCtx( } } } else if (msg.type === 'function_call') { - parts.push({ + const functionCallPart: Record = { functionCall: { id: msg.callId, name: msg.name, args: JSON.parse(msg.args || '{}'), }, - }); + }; + if (msg.thoughtSignature) { + functionCallPart.thoughtSignature = msg.thoughtSignature; + } + parts.push(functionCallPart); } else if (msg.type === 'function_call_output') { const response = msg.isError ? { error: msg.output } : { output: msg.output }; parts.push({ diff --git a/agents/src/voice/generation.ts b/agents/src/voice/generation.ts index 82777ce12..82d95849f 100644 --- a/agents/src/voice/generation.ts +++ b/agents/src/voice/generation.ts @@ -442,6 +442,8 @@ export function performLLMInference( callId: `${data.id}/fnc_${data.generatedToolCalls.length}`, name: tool.name, args: tool.args, + // Preserve thought signature for Gemini 3+ thinking mode + thoughtSignature: tool.thoughtSignature, }); data.generatedToolCalls.push(toolCall); diff --git a/plugins/google/package.json b/plugins/google/package.json index 5890d23e5..67ef0c37c 100644 --- a/plugins/google/package.json +++ b/plugins/google/package.json @@ -43,7 +43,7 @@ "typescript": "^5.0.0" }, "dependencies": { - "@google/genai": "^1.13.0", + "@google/genai": "^1.34.0", "@livekit/mutex": "^1.1.1", "@types/json-schema": "^7.0.15", "json-schema": "^0.4.0" diff --git a/plugins/google/src/llm.ts b/plugins/google/src/llm.ts index 48f74a1fa..990a953b7 100644 --- a/plugins/google/src/llm.ts +++ b/plugins/google/src/llm.ts @@ -449,6 +449,8 @@ export class LLMStream extends llm.LLMStream { callId: part.functionCall.id || shortuuid('function_call_'), name: part.functionCall.name!, args: JSON.stringify(part.functionCall.args!), + // Preserve thought signature for Gemini 3+ thinking mode + thoughtSignature: part.thoughtSignature, }), ], }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 39b53189a..3e29b979f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -468,8 +468,8 @@ importers: plugins/google: dependencies: '@google/genai': - specifier: ^1.13.0 - version: 1.13.0 + specifier: ^1.34.0 + version: 1.34.0 '@livekit/mutex': specifier: ^1.1.1 version: 1.1.1 @@ -1402,11 +1402,11 @@ packages: cpu: [x64] os: [win32] - '@google/genai@1.13.0': - resolution: {integrity: sha512-BxilXzE8cJ0zt5/lXk6KwuBcIT9P2Lbi2WXhwWMbxf1RNeC68/8DmYQqMrzQP333CieRMdbDXs0eNCphLoScWg==} + '@google/genai@1.34.0': + resolution: {integrity: sha512-vu53UMPvjmb7PGzlYu6Tzxso8Dfhn+a7eQFaS2uNemVtDZKwzSpJ5+ikqBbXplF7RGB1STcVDqCkPvquiwb2sw==} engines: {node: '>=20.0.0'} peerDependencies: - '@modelcontextprotocol/sdk': ^1.11.0 + '@modelcontextprotocol/sdk': ^1.24.0 peerDependenciesMeta: '@modelcontextprotocol/sdk': optional: true @@ -2657,6 +2657,10 @@ packages: damerau-levenshtein@1.0.8: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + data-view-buffer@1.0.1: resolution: {integrity: sha512-0lht7OugA5x3iJLOWFhWK/5ehONdprk0ISXqVFn/NFrDu+cuc8iADFrGQz5BnRK7LLU3JmkbXSxaqX+/mXYtUA==} engines: {node: '>= 0.4'} @@ -3124,6 +3128,10 @@ packages: picomatch: optional: true + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + file-entry-cache@6.0.1: resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} engines: {node: ^10.12.0 || >=12.0.0} @@ -3170,6 +3178,10 @@ packages: resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} engines: {node: '>= 6'} + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + fs-extra@7.0.1: resolution: {integrity: sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==} engines: {node: '>=6 <7 || >=8'} @@ -3196,13 +3208,13 @@ packages: functions-have-names@1.2.3: resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==} - gaxios@6.7.1: - resolution: {integrity: sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==} - engines: {node: '>=14'} + gaxios@7.1.3: + resolution: {integrity: sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==} + engines: {node: '>=18'} - gcp-metadata@6.1.1: - resolution: {integrity: sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==} - engines: {node: '>=14'} + gcp-metadata@8.1.2: + resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} + engines: {node: '>=18'} get-func-name@2.0.2: resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==} @@ -3278,12 +3290,12 @@ packages: resolution: {integrity: sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==} engines: {node: '>=10'} - google-auth-library@9.15.1: - resolution: {integrity: sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==} - engines: {node: '>=14'} + google-auth-library@10.5.0: + resolution: {integrity: sha512-7ABviyMOlX5hIVD60YOfHw4/CxOfBhyduaYB+wbFWCWoni4N7SLcV46hrVRktuBbZjFC9ONyqamZITN7q3n32w==} + engines: {node: '>=18'} - google-logging-utils@0.0.2: - resolution: {integrity: sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==} + google-logging-utils@1.1.3: + resolution: {integrity: sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==} engines: {node: '>=14'} gopd@1.0.1: @@ -3299,9 +3311,9 @@ packages: graphemer@1.4.0: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} - gtoken@7.1.0: - resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==} - engines: {node: '>=14.0.0'} + gtoken@8.0.0: + resolution: {integrity: sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==} + engines: {node: '>=18'} guid-typescript@1.0.9: resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==} @@ -3491,10 +3503,6 @@ packages: resolution: {integrity: sha512-nA2hv5XIhLR3uVzDDfCIknerhx8XUKnstuOERPNNIinXG7v9u+ohXF67vxm4TPTEPU6lm61ZkwP3c9PCB97rhg==} engines: {node: '>= 0.4'} - is-stream@2.0.1: - resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} - engines: {node: '>=8'} - is-stream@3.0.0: resolution: {integrity: sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -3846,6 +3854,11 @@ packages: natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + node-fetch@2.7.0: resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} engines: {node: 4.x || >=6.0.0} @@ -3855,6 +3868,10 @@ packages: encoding: optional: true + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + npm-run-path@5.3.0: resolution: {integrity: sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -4717,10 +4734,6 @@ packages: resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} hasBin: true - uuid@9.0.1: - resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} - hasBin: true - validator@13.12.0: resolution: {integrity: sha512-c1Q0mCiPlgdTVVVIJIrBuxNicYE+t/7oKeI9MWLj3fh/uq2Pxh/3eeWbVZ4OcGW1TUf53At0njHw5SMdA3tmMg==} engines: {node: '>= 0.10'} @@ -4826,6 +4839,10 @@ packages: resolution: {integrity: sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==} engines: {node: '>=10.13.0'} + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -5484,13 +5501,12 @@ snapshots: '@ffmpeg-installer/win32-x64@4.1.0': optional: true - '@google/genai@1.13.0': + '@google/genai@1.34.0': dependencies: - google-auth-library: 9.15.1 + google-auth-library: 10.5.0 ws: 8.18.3 transitivePeerDependencies: - bufferutil - - encoding - supports-color - utf-8-validate @@ -6772,6 +6788,8 @@ snapshots: damerau-levenshtein@1.0.8: {} + data-uri-to-buffer@4.0.1: {} + data-view-buffer@1.0.1: dependencies: call-bind: 1.0.7 @@ -7411,6 +7429,11 @@ snapshots: optionalDependencies: picomatch: 4.0.2 + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + file-entry-cache@6.0.1: dependencies: flat-cache: 3.2.0 @@ -7465,6 +7488,10 @@ snapshots: hasown: 2.0.2 mime-types: 2.1.35 + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + fs-extra@7.0.1: dependencies: graceful-fs: 4.2.11 @@ -7493,24 +7520,21 @@ snapshots: functions-have-names@1.2.3: {} - gaxios@6.7.1: + gaxios@7.1.3: dependencies: extend: 3.0.2 https-proxy-agent: 7.0.6 - is-stream: 2.0.1 - node-fetch: 2.7.0 - uuid: 9.0.1 + node-fetch: 3.3.2 + rimraf: 5.0.10 transitivePeerDependencies: - - encoding - supports-color - gcp-metadata@6.1.1: + gcp-metadata@8.1.2: dependencies: - gaxios: 6.7.1 - google-logging-utils: 0.0.2 + gaxios: 7.1.3 + google-logging-utils: 1.1.3 json-bigint: 1.0.0 transitivePeerDependencies: - - encoding - supports-color get-func-name@2.0.2: {} @@ -7620,19 +7644,19 @@ snapshots: merge2: 1.4.1 slash: 3.0.0 - google-auth-library@9.15.1: + google-auth-library@10.5.0: dependencies: base64-js: 1.5.1 ecdsa-sig-formatter: 1.0.11 - gaxios: 6.7.1 - gcp-metadata: 6.1.1 - gtoken: 7.1.0 + gaxios: 7.1.3 + gcp-metadata: 8.1.2 + google-logging-utils: 1.1.3 + gtoken: 8.0.0 jws: 4.0.0 transitivePeerDependencies: - - encoding - supports-color - google-logging-utils@0.0.2: {} + google-logging-utils@1.1.3: {} gopd@1.0.1: dependencies: @@ -7644,12 +7668,11 @@ snapshots: graphemer@1.4.0: {} - gtoken@7.1.0: + gtoken@8.0.0: dependencies: - gaxios: 6.7.1 + gaxios: 7.1.3 jws: 4.0.0 transitivePeerDependencies: - - encoding - supports-color guid-typescript@1.0.9: {} @@ -7811,8 +7834,6 @@ snapshots: dependencies: call-bind: 1.0.7 - is-stream@2.0.1: {} - is-stream@3.0.0: {} is-string@1.0.7: @@ -8142,10 +8163,18 @@ snapshots: natural-compare@1.4.0: {} + node-domexception@1.0.0: {} + node-fetch@2.7.0: dependencies: whatwg-url: 5.0.0 + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + npm-run-path@5.3.0: dependencies: path-key: 4.0.0 @@ -9114,8 +9143,6 @@ snapshots: uuid@11.1.0: {} - uuid@9.0.1: {} - validator@13.12.0: {} vite-node@1.6.0(@types/node@22.15.30): @@ -9240,6 +9267,8 @@ snapshots: glob-to-regexp: 0.4.1 graceful-fs: 4.2.11 + web-streams-polyfill@3.3.3: {} + webidl-conversions@3.0.1: {} webidl-conversions@4.0.2: {} From 3043ebfa2668555ba818aaa55ff2b614f226f6d3 Mon Sep 17 00:00:00 2001 From: Tom C Date: Mon, 5 Jan 2026 14:45:31 +1000 Subject: [PATCH 2/3] chore: add changeset for google plugin patch --- .changeset/strong-hoops-lick.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/strong-hoops-lick.md diff --git a/.changeset/strong-hoops-lick.md b/.changeset/strong-hoops-lick.md new file mode 100644 index 000000000..6c28aa569 --- /dev/null +++ b/.changeset/strong-hoops-lick.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-google': minor +--- + +preserve thought signatures for Gemini 3+ function calling From 566886067b708b7a843f79459dfcd91a665e6162 Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 6 Jan 2026 09:20:39 +1000 Subject: [PATCH 3/3] chore: change changeset to patch instead of minor --- .changeset/strong-hoops-lick.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/strong-hoops-lick.md b/.changeset/strong-hoops-lick.md index 6c28aa569..24b3cf367 100644 --- a/.changeset/strong-hoops-lick.md +++ b/.changeset/strong-hoops-lick.md @@ -1,5 +1,5 @@ --- -'@livekit/agents-plugin-google': minor +'@livekit/agents-plugin-google': patch --- preserve thought signatures for Gemini 3+ function calling