[smg] cleanup router RAII guards (#16560)

This commit is contained in:
fzyzcjy
2026-01-08 16:39:53 -08:00
committed by GitHub
parent 1bc7aa5801
commit 9e3a032ad6
9 changed files with 79 additions and 131 deletions
@@ -835,7 +835,7 @@ impl PDRouter {
prefill: Arc<dyn Worker>,
decode: Arc<dyn Worker>,
) -> Response {
use crate::core::attach_guards_to_response;
use crate::core::AttachedBody;
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
@@ -885,7 +885,7 @@ impl PDRouter {
// Attach load guards to response body for proper RAII lifecycle
// Guards are dropped when response body is consumed or client disconnects
let guards = vec![WorkerLoadGuard::new(prefill), WorkerLoadGuard::new(decode)];
attach_guards_to_response(guards, response)
AttachedBody::wrap_response(response, guards)
}
// Helper to process non-streaming decode response with logprob merging
+2 -2
View File
@@ -16,7 +16,7 @@ use crate::{
app_context::AppContext,
config::types::RetryConfig,
core::{
is_retryable_status, ConnectionMode, RetryExecutor, Worker, WorkerLoadGuard,
is_retryable_status, AttachedBody, ConnectionMode, RetryExecutor, Worker, WorkerLoadGuard,
WorkerRegistry, WorkerType, UNKNOWN_MODEL_ID,
},
observability::{
@@ -629,7 +629,7 @@ impl Router {
// Attach load guard to response body for proper RAII lifecycle
// Guard is dropped when response body is consumed or client disconnects
if let Some(guard) = load_guard {
response = guard.attach_to_response(response);
response = AttachedBody::wrap_response(response, guard);
}
response
}