iceBear67
c48b60d383
Main turns now set prompt_cache_key to the session/conv id. Responses already sent that via the conv-id fallback; Chat Completions now puts the same value in `user`. Recap and /btw keep sharing the parent session key — not an agent-scoped suffix.
135 lines
6.8 KiB
Diff
135 lines
6.8 KiB
Diff
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
From: iceBear67 <icebear67@sfclub.cc>
|
|
Date: Sun, 16 Aug 2026 03:13:22 +0000
|
|
Subject: [PATCH] Pin a session-stable prompt_cache_key on main turns
|
|
|
|
Main turns already reached the Responses wire as prompt_cache_key via
|
|
the x_grok_conv_id fallback. Set the field explicitly so Chat Completions
|
|
can put the same session id in `user`, and so recap /btw keep sharing
|
|
one key. Do not agent-scope: that would split the prefix those side
|
|
calls are built to ride.
|
|
|
|
diff --git a/crates/codegen/xai-chat-state/src/actor/request_builder.rs b/crates/codegen/xai-chat-state/src/actor/request_builder.rs
|
|
index 4d9ce755a316ab0cca430aa3126d3ceb18e541b5..7c787b6dce09682445438274ed5a35008f242a6d 100644
|
|
--- a/crates/codegen/xai-chat-state/src/actor/request_builder.rs
|
|
+++ b/crates/codegen/xai-chat-state/src/actor/request_builder.rs
|
|
@@ -105,7 +105,10 @@ impl ChatStateActor {
|
|
x_grok_deployment_id: None,
|
|
x_grok_user_id: None,
|
|
trace,
|
|
- prompt_cache_key: None,
|
|
+ // Same session id the Responses mapping already fell back to via
|
|
+ // `x_grok_conv_id`. Set it explicitly so Chat Completions `user`
|
|
+ // and side-calls share one key without depending on that fallback.
|
|
+ prompt_cache_key: Some(conv_id.clone()),
|
|
reasoning_effort: self.state.sampling_config.reasoning_effort,
|
|
json_schema: None,
|
|
}
|
|
diff --git a/crates/codegen/xai-chat-state/src/actor/tests.rs b/crates/codegen/xai-chat-state/src/actor/tests.rs
|
|
index 604a121b4897d1e245a15f99318130af0ad02462..e9899af98ecf80eb541ce120f4aad489e0e25dae 100644
|
|
--- a/crates/codegen/xai-chat-state/src/actor/tests.rs
|
|
+++ b/crates/codegen/xai-chat-state/src/actor/tests.rs
|
|
@@ -1576,6 +1576,7 @@ async fn build_request_includes_all_messages() {
|
|
assert_eq!(request.items.len(), 2);
|
|
assert_eq!(request.x_grok_conv_id, Some("conv-1".to_string()));
|
|
assert_eq!(request.x_grok_req_id, Some("req-1".to_string()));
|
|
+ assert_eq!(request.prompt_cache_key, Some("conv-1".to_string()));
|
|
}
|
|
|
|
#[tokio::test]
|
|
diff --git a/crates/codegen/xai-grok-sampling-types/src/conversation.rs b/crates/codegen/xai-grok-sampling-types/src/conversation.rs
|
|
index 982839bbe822874651bc37e257e4e4a7442f6f87..91d61f3a46524cdf5c11cb753c739abe8481f503 100644
|
|
--- a/crates/codegen/xai-grok-sampling-types/src/conversation.rs
|
|
+++ b/crates/codegen/xai-grok-sampling-types/src/conversation.rs
|
|
@@ -623,10 +623,29 @@ pub struct ConversationRequest {
|
|
/// JSON Schema for structured output (strict mode).
|
|
pub json_schema: Option<serde_json::Value>,
|
|
/// Sticky routing key for prompt-cache reuse; overrides `x_grok_conv_id` for routing.
|
|
+ ///
|
|
+ /// Session-scoped, not agent-scoped: recap and `/btw` replay the parent
|
|
+ /// conversation under this key. A per-agent suffix would split the prefix
|
|
+ /// cache those side-calls are built to share. Only the Responses mapping
|
|
+ /// puts this field on the wire; Chat Completions surfaces the same value
|
|
+ /// as `user` (see [`Self::session_cache_key`]).
|
|
pub prompt_cache_key: Option<String>,
|
|
}
|
|
|
|
impl ConversationRequest {
|
|
+ /// Session-stable key for prompt-cache sticky routing.
|
|
+ ///
|
|
+ /// Prefer an explicit [`Self::prompt_cache_key`], then session id, then
|
|
+ /// conv id. Callers that share a conversation prefix (main turn, recap,
|
|
+ /// `/btw`) must resolve to the same string.
|
|
+ pub fn session_cache_key(&self) -> Option<&str> {
|
|
+ self.prompt_cache_key
|
|
+ .as_deref()
|
|
+ .or(self.x_grok_session_id.as_deref())
|
|
+ .or(self.x_grok_conv_id.as_deref())
|
|
+ .filter(|s| !s.is_empty())
|
|
+ }
|
|
+
|
|
/// Strip every image; returns the stripped URLs.
|
|
pub fn strip_images(&mut self) -> Vec<Arc<str>> {
|
|
strip_images_where(&mut self.items, |_| true)
|
|
@@ -2399,6 +2418,35 @@ mod tests {
|
|
use crate::tool_overrides::*;
|
|
use assert_matches::assert_matches;
|
|
|
|
+ #[test]
|
|
+ fn session_cache_key_prefers_explicit_then_session_then_conv() {
|
|
+ let mut req = ConversationRequest {
|
|
+ x_grok_conv_id: Some("conv".into()),
|
|
+ x_grok_session_id: Some("session".into()),
|
|
+ prompt_cache_key: Some("explicit".into()),
|
|
+ ..Default::default()
|
|
+ };
|
|
+ assert_eq!(req.session_cache_key(), Some("explicit"));
|
|
+ req.prompt_cache_key = None;
|
|
+ assert_eq!(req.session_cache_key(), Some("session"));
|
|
+ req.x_grok_session_id = None;
|
|
+ assert_eq!(req.session_cache_key(), Some("conv"));
|
|
+ req.x_grok_conv_id = Some(String::new());
|
|
+ assert_eq!(req.session_cache_key(), None);
|
|
+ }
|
|
+
|
|
+ #[test]
|
|
+ fn chat_completions_user_carries_session_cache_key() {
|
|
+ let req = ConversationRequest {
|
|
+ items: vec![ConversationItem::user("hi")],
|
|
+ model: Some("test-model".into()),
|
|
+ prompt_cache_key: Some("sess-1".into()),
|
|
+ ..Default::default()
|
|
+ };
|
|
+ let mapped = ChatCompletionRequest::from(req);
|
|
+ assert_eq!(mapped.user.as_deref(), Some("sess-1"));
|
|
+ }
|
|
+
|
|
/// Keeps `forwards_prompt_cache_key()` honest against each mapping: a key that never reaches the wire looks like a 0% cache hit, not a bug.
|
|
#[test]
|
|
fn prompt_cache_key_reaches_the_wire_only_where_the_backend_claims() {
|
|
diff --git a/crates/codegen/xai-grok-sampling-types/src/conversation/chat_completions.rs b/crates/codegen/xai-grok-sampling-types/src/conversation/chat_completions.rs
|
|
index dac77d82f7f96836f5f4ed26f2e12eb13164e4f5..b80820fbbf3c96e348f44880c432361eb537817a 100644
|
|
--- a/crates/codegen/xai-grok-sampling-types/src/conversation/chat_completions.rs
|
|
+++ b/crates/codegen/xai-grok-sampling-types/src/conversation/chat_completions.rs
|
|
@@ -295,7 +295,7 @@ impl From<ConversationRequest> for ChatCompletionRequest {
|
|
top_p: req.top_p,
|
|
frequency_penalty: None,
|
|
presence_penalty: None,
|
|
- user: None,
|
|
+ user: req.session_cache_key().map(str::to_owned),
|
|
tools,
|
|
tool_choice,
|
|
search_parameters: None,
|
|
diff --git a/crates/codegen/xai-grok-sampling-types/src/conversation/chat_completions_tests.rs b/crates/codegen/xai-grok-sampling-types/src/conversation/chat_completions_tests.rs
|
|
index 8d2afa36362471676804bfa8676d0294ca56049d..1a29113ca0e467b9a5e39faa06cd468d4f45a242 100644
|
|
--- a/crates/codegen/xai-grok-sampling-types/src/conversation/chat_completions_tests.rs
|
|
+++ b/crates/codegen/xai-grok-sampling-types/src/conversation/chat_completions_tests.rs
|
|
@@ -52,6 +52,7 @@ fn test_conversation_request_to_chat_completion() {
|
|
assert_eq!(chat_req.model, Some("grok-3".to_string()));
|
|
assert_eq!(chat_req.temperature, Some(0.7));
|
|
assert_eq!(chat_req.messages.len(), 2);
|
|
+ assert_eq!(chat_req.user, None);
|
|
}
|
|
|
|
#[test]
|