From b27a5e9248dd0fd01bbad55c58cef0ac2fa0bd43 Mon Sep 17 00:00:00 2001 From: Florian Hotze Date: Thu, 2 Jul 2026 21:04:46 +0200 Subject: [PATCH] [gemini] Fix HTTP 400 when using tool calling with thinking models (#21097) * [gemini] Fix HTTP 400 when using tool calling with thinking models Signed-off-by: Florian Hotze --- .../gemini/internal/api/GeminiApiClient.java | 38 +++++++++----- .../internal/api/GeminiLLMToolCall.java | 51 +++++++++++++++++++ .../internal/api/dto/GeminiFunctionCall.java | 2 +- .../gemini/internal/api/dto/GeminiPart.java | 3 +- .../gemini/internal/hli/GeminiHLIService.java | 6 +-- 5 files changed, 82 insertions(+), 18 deletions(-) create mode 100644 bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/api/GeminiLLMToolCall.java diff --git a/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/api/GeminiApiClient.java b/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/api/GeminiApiClient.java index 8ad02e67c1..3cd89e9cae 100644 --- a/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/api/GeminiApiClient.java +++ b/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/api/GeminiApiClient.java @@ -51,7 +51,6 @@ import org.openhab.binding.gemini.internal.api.dto.response.GeminiModelsResponse import org.openhab.binding.gemini.internal.api.dto.response.GeminiResponse; import org.openhab.core.voice.text.conversation.Conversation; import org.openhab.core.voice.text.interpreter.llm.LLMTool; -import org.openhab.core.voice.text.interpreter.llm.LLMToolCall; import org.openhab.core.voice.text.interpreter.llm.LLMToolParam; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -105,7 +104,7 @@ public class GeminiApiClient { GeminiContent systemInstruction = createSystemInstruction(systemMessage); // Contents - GeminiPart userPart = new GeminiPart(prompt, null, null, null); + GeminiPart userPart = new GeminiPart(prompt, null, null, null, null); GeminiContent userContent = new GeminiContent(ROLE_USER, List.of(userPart)); // Config @@ -141,21 +140,21 @@ public class GeminiApiClient { for (Conversation.Message msg : history) { switch (msg.role()) { case USER: { - GeminiPart part = new GeminiPart(msg.content(), null, null, null); - contents.add(new GeminiContent(ROLE_USER, List.of(part))); + GeminiPart part = new GeminiPart(msg.content(), null, null, null, null); + contents.add(new GeminiContent(ROLE_USER, new ArrayList<>(List.of(part)))); break; } case OPENHAB: { - GeminiPart part = new GeminiPart(msg.content(), null, null, null); - contents.add(new GeminiContent(ROLE_MODEL, List.of(part))); + GeminiPart part = new GeminiPart(msg.content(), null, null, null, null); + contents.add(new GeminiContent(ROLE_MODEL, new ArrayList<>(List.of(part)))); break; } case TOOL_CALL: { - LLMToolCall toolCall = LLMToolCall.fromJson(msg.content()); - String name = toolCall.tool().replaceAll("[^a-zA-Z0-9_-]", "_"); + GeminiLLMToolCall toolCall = GeminiLLMToolCall.fromJson(msg.content()); + String name = toolCall.tool.replaceAll("[^a-zA-Z0-9_-]", "_"); pendingToolCallNames.add(name); - GeminiFunctionCall fc = new GeminiFunctionCall(name, toolCall.params()); - GeminiPart part = new GeminiPart(null, fc, null, null); + GeminiFunctionCall fc = new GeminiFunctionCall(name, toolCall.params, toolCall.id); + GeminiPart part = new GeminiPart(null, fc, null, null, toolCall.thoughtSignature); contents.add(new GeminiContent(ROLE_MODEL, List.of(part))); break; } @@ -166,8 +165,21 @@ public class GeminiApiClient { break; // TOOL_RETURN without preceding TOOL_CALL - ignore } GeminiFunctionResponse fr = new GeminiFunctionResponse(name, Map.of("result", msg.content())); - GeminiPart part = new GeminiPart(null, null, fr, null); - contents.add(new GeminiContent(ROLE_USER, List.of(part))); + GeminiPart part = new GeminiPart(null, null, fr, null, null); + + // Consolidate consecutive TOOL_RETURNs into the same ROLE_USER content + if (!contents.isEmpty() && ROLE_USER.equals(contents.getLast().role())) { + GeminiContent lastContent = contents.getLast(); + List lastParts = lastContent.parts(); + if (lastParts != null && !lastParts.isEmpty() + && lastParts.getFirst().functionResponse() != null) { + lastParts.add(part); + } else { + contents.add(new GeminiContent(ROLE_USER, new ArrayList<>(List.of(part)))); + } + } else { + contents.add(new GeminiContent(ROLE_USER, new ArrayList<>(List.of(part)))); + } break; } case THINKING: @@ -356,7 +368,7 @@ public class GeminiApiClient { private @Nullable GeminiContent createSystemInstruction(@Nullable String systemMessage) { if (systemMessage != null && !systemMessage.isBlank()) { - GeminiPart sysPart = new GeminiPart(systemMessage, null, null, null); + GeminiPart sysPart = new GeminiPart(systemMessage, null, null, null, null); return new GeminiContent(null, List.of(sysPart)); } return null; diff --git a/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/api/GeminiLLMToolCall.java b/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/api/GeminiLLMToolCall.java new file mode 100644 index 0000000000..d3aabf2c97 --- /dev/null +++ b/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/api/GeminiLLMToolCall.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.binding.gemini.internal.api; + +import java.util.Map; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.core.voice.text.interpreter.llm.LLMToolCall; + +import com.google.gson.JsonSyntaxException; + +/** + * A DTO to store information about a Gemini tool call. + * Extends {@link LLMToolCall} with additional fields required by the Gemini API. + * + * @author Florian Hotze - Initial contribution + */ +@NonNullByDefault +public class GeminiLLMToolCall extends LLMToolCall { + public final @Nullable String id; + public final @Nullable String thoughtSignature; + + public GeminiLLMToolCall(String tool, Map params, @Nullable String id, + @Nullable String thoughtSignature) { + super(tool, params); + this.id = id; + this.thoughtSignature = thoughtSignature; + } + + public static GeminiLLMToolCall fromJson(String json) throws JsonSyntaxException { + GeminiLLMToolCall call = GSON.fromJson(json, GeminiLLMToolCall.class); + if (call == null) { + throw new JsonSyntaxException("Deserialized GeminiLLMToolCall is null."); + } + if (call.tool == null || call.params == null) { + throw new JsonSyntaxException("Deserialized GeminiLLMToolCall has null tool or params."); + } + return call; + } +} diff --git a/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/api/dto/GeminiFunctionCall.java b/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/api/dto/GeminiFunctionCall.java index a0d3bd731a..68eefa16b0 100644 --- a/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/api/dto/GeminiFunctionCall.java +++ b/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/api/dto/GeminiFunctionCall.java @@ -27,5 +27,5 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; */ @JsonIgnoreProperties(ignoreUnknown = true) @NonNullByDefault -public record GeminiFunctionCall(@Nullable String name, @Nullable Map args) { +public record GeminiFunctionCall(@Nullable String name, @Nullable Map args, @Nullable String id) { } diff --git a/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/api/dto/GeminiPart.java b/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/api/dto/GeminiPart.java index 85fe602455..0ae41c0697 100644 --- a/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/api/dto/GeminiPart.java +++ b/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/api/dto/GeminiPart.java @@ -27,5 +27,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @JsonIgnoreProperties(ignoreUnknown = true) @NonNullByDefault public record GeminiPart(@Nullable String text, @Nullable GeminiFunctionCall functionCall, - @Nullable GeminiFunctionResponse functionResponse, @Nullable Boolean thought) { + @Nullable GeminiFunctionResponse functionResponse, @Nullable Boolean thought, + @Nullable String thoughtSignature) { } diff --git a/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/hli/GeminiHLIService.java b/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/hli/GeminiHLIService.java index 32f3e9ca34..a9d275d4b3 100644 --- a/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/hli/GeminiHLIService.java +++ b/bundles/org.openhab.binding.gemini/src/main/java/org/openhab/binding/gemini/internal/hli/GeminiHLIService.java @@ -28,6 +28,7 @@ import org.openhab.binding.gemini.internal.GeminiConfiguration; import org.openhab.binding.gemini.internal.GeminiHandler; import org.openhab.binding.gemini.internal.api.GeminiApiClient; import org.openhab.binding.gemini.internal.api.GeminiApiException; +import org.openhab.binding.gemini.internal.api.GeminiLLMToolCall; import org.openhab.binding.gemini.internal.api.dto.GeminiContent; import org.openhab.binding.gemini.internal.api.dto.GeminiFunctionCall; import org.openhab.binding.gemini.internal.api.dto.GeminiPart; @@ -43,7 +44,6 @@ import org.openhab.core.voice.text.conversation.Conversation; import org.openhab.core.voice.text.conversation.ConversationException; import org.openhab.core.voice.text.conversation.ConversationRole; import org.openhab.core.voice.text.interpreter.llm.LLMTool; -import org.openhab.core.voice.text.interpreter.llm.LLMToolCall; import org.openhab.core.voice.text.interpreter.llm.LLMToolException; import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; @@ -252,8 +252,8 @@ public class GeminiHLIService implements ThingHandlerService, HumanLanguageInter String toolName = fc.name(); Map args = fc.args(); - LLMToolCall llmToolCall = new LLMToolCall(toolName != null ? toolName : "", - args != null ? args : new HashMap<>()); + GeminiLLMToolCall llmToolCall = new GeminiLLMToolCall(toolName != null ? toolName : "", + args != null ? args : new HashMap<>(), fc.id(), part.thoughtSignature()); try { conversation.addMessage(ConversationRole.TOOL_CALL, llmToolCall.toJson()); } catch (ConversationException e) {