From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: iceBear67 Date: Sun, 16 Aug 2026 03:13:22 +0000 Subject: [PATCH] Pin a session-stable prompt_cache_key on main turns Main turns already reached the Responses wire as prompt_cache_key via the x_grok_conv_id fallback. Set the field explicitly so Chat Completions can put the same session id in `user`, and so recap /btw keep sharing one key. Do not agent-scope: that would split the prefix those side calls are built to ride. diff --git a/crates/codegen/xai-chat-state/src/actor/request_builder.rs b/crates/codegen/xai-chat-state/src/actor/request_builder.rs index 64f497a0b0419b167faea18e35b32b941080620e..109a97521e590dc1f14d367c3d349600001558f8 100644 --- a/crates/codegen/xai-chat-state/src/actor/request_builder.rs +++ b/crates/codegen/xai-chat-state/src/actor/request_builder.rs @@ -106,7 +106,10 @@ impl ChatStateActor { x_grok_deployment_id: None, x_grok_user_id: None, trace, - prompt_cache_key: None, + // Same session id the Responses mapping already fell back to via + // `x_grok_conv_id`. Set it explicitly so Chat Completions `user` + // and side-calls share one key without depending on that fallback. + prompt_cache_key: Some(conv_id.clone()), reasoning_effort: self.state.sampling_config.reasoning_effort, json_schema: None, } diff --git a/crates/codegen/xai-chat-state/src/actor/tests.rs b/crates/codegen/xai-chat-state/src/actor/tests.rs index 07d581d48e78a7c69e78a7f0d8d2ce1844a3190b..dee7b542755b8d97dfcc4e621d01bb809488d017 100644 --- a/crates/codegen/xai-chat-state/src/actor/tests.rs +++ b/crates/codegen/xai-chat-state/src/actor/tests.rs @@ -1576,6 +1576,7 @@ async fn build_request_includes_all_messages() { assert_eq!(request.items.len(), 2); assert_eq!(request.x_grok_conv_id, Some("conv-1".to_string())); assert_eq!(request.x_grok_req_id, Some("req-1".to_string())); + assert_eq!(request.prompt_cache_key, Some("conv-1".to_string())); } #[tokio::test] diff --git a/crates/codegen/xai-grok-sampling-types/src/conversation.rs b/crates/codegen/xai-grok-sampling-types/src/conversation.rs index 7bd0cd870c5400ef4d3fb2ba6ce50cfc53ffd62c..db582daced370d89cc1158d5cbbf81b7a156ba4f 100644 --- a/crates/codegen/xai-grok-sampling-types/src/conversation.rs +++ b/crates/codegen/xai-grok-sampling-types/src/conversation.rs @@ -627,10 +627,29 @@ pub struct ConversationRequest { /// JSON Schema for structured output (strict mode). pub json_schema: Option, /// Sticky routing key for prompt-cache reuse; overrides `x_grok_conv_id` for routing. + /// + /// Session-scoped, not agent-scoped: recap and `/btw` replay the parent + /// conversation under this key. A per-agent suffix would split the prefix + /// cache those side-calls are built to share. Only the Responses mapping + /// puts this field on the wire; Chat Completions surfaces the same value + /// as `user` (see [`Self::session_cache_key`]). pub prompt_cache_key: Option, } impl ConversationRequest { + /// Session-stable key for prompt-cache sticky routing. + /// + /// Prefer an explicit [`Self::prompt_cache_key`], then session id, then + /// conv id. Callers that share a conversation prefix (main turn, recap, + /// `/btw`) must resolve to the same string. + pub fn session_cache_key(&self) -> Option<&str> { + self.prompt_cache_key + .as_deref() + .or(self.x_grok_session_id.as_deref()) + .or(self.x_grok_conv_id.as_deref()) + .filter(|s| !s.is_empty()) + } + /// Strip every image; returns the stripped URLs. pub fn strip_images(&mut self) -> Vec> { strip_images_where(&mut self.items, |_| true) @@ -2416,6 +2435,35 @@ mod tests { use crate::tool_overrides::*; use assert_matches::assert_matches; + #[test] + fn session_cache_key_prefers_explicit_then_session_then_conv() { + let mut req = ConversationRequest { + x_grok_conv_id: Some("conv".into()), + x_grok_session_id: Some("session".into()), + prompt_cache_key: Some("explicit".into()), + ..Default::default() + }; + assert_eq!(req.session_cache_key(), Some("explicit")); + req.prompt_cache_key = None; + assert_eq!(req.session_cache_key(), Some("session")); + req.x_grok_session_id = None; + assert_eq!(req.session_cache_key(), Some("conv")); + req.x_grok_conv_id = Some(String::new()); + assert_eq!(req.session_cache_key(), None); + } + + #[test] + fn chat_completions_user_carries_session_cache_key() { + let req = ConversationRequest { + items: vec![ConversationItem::user("hi")], + model: Some("test-model".into()), + prompt_cache_key: Some("sess-1".into()), + ..Default::default() + }; + let mapped = ChatCompletionRequest::from(req); + assert_eq!(mapped.user.as_deref(), Some("sess-1")); + } + /// Keeps `forwards_prompt_cache_key()` honest against each mapping: a key that never reaches the wire looks like a 0% cache hit, not a bug. #[test] fn prompt_cache_key_reaches_the_wire_only_where_the_backend_claims() { diff --git a/crates/codegen/xai-grok-sampling-types/src/conversation/chat_completions.rs b/crates/codegen/xai-grok-sampling-types/src/conversation/chat_completions.rs index dac77d82f7f96836f5f4ed26f2e12eb13164e4f5..05a683317ae774f3ba674202ff12ae3ae50f56ec 100644 --- a/crates/codegen/xai-grok-sampling-types/src/conversation/chat_completions.rs +++ b/crates/codegen/xai-grok-sampling-types/src/conversation/chat_completions.rs @@ -251,6 +251,8 @@ impl From for ConversationItem { impl From for ChatCompletionRequest { fn from(req: ConversationRequest) -> Self { + // Copy before any field moves — `session_cache_key` borrows `req`. + let user = req.session_cache_key().map(str::to_owned); let messages: Vec = conversation_to_chat_messages(req.items); let tools_is_empty = req.tools.is_empty(); @@ -295,7 +297,7 @@ impl From for ChatCompletionRequest { top_p: req.top_p, frequency_penalty: None, presence_penalty: None, - user: None, + user, tools, tool_choice, search_parameters: None, diff --git a/crates/codegen/xai-grok-sampling-types/src/conversation/chat_completions_tests.rs b/crates/codegen/xai-grok-sampling-types/src/conversation/chat_completions_tests.rs index 8d2afa36362471676804bfa8676d0294ca56049d..1a29113ca0e467b9a5e39faa06cd468d4f45a242 100644 --- a/crates/codegen/xai-grok-sampling-types/src/conversation/chat_completions_tests.rs +++ b/crates/codegen/xai-grok-sampling-types/src/conversation/chat_completions_tests.rs @@ -52,6 +52,7 @@ fn test_conversation_request_to_chat_completion() { assert_eq!(chat_req.model, Some("grok-3".to_string())); assert_eq!(chat_req.temperature, Some(0.7)); assert_eq!(chat_req.messages.len(), 2); + assert_eq!(chat_req.user, None); } #[test]