[model-gateway] feat: add DAG parallel execution support and workflow optimization (#14999)

This commit is contained in:
Simo Lin
2025-12-12 08:31:12 -08:00
committed by GitHub
parent 10c68f6236
commit 306e5b8d0b
13 changed files with 768 additions and 193 deletions
+89 -1
View File
@@ -1,6 +1,6 @@
//! Workflow definition types
use std::{sync::Arc, time::Duration};
use std::{collections::HashSet, sync::Arc, time::Duration};
use super::{
executor::StepExecutor,
@@ -15,6 +15,7 @@ pub struct StepDefinition {
pub retry_policy: Option<RetryPolicy>,
pub timeout: Option<Duration>,
pub on_failure: FailureAction,
pub depends_on: Vec<StepId>,
}
impl StepDefinition {
@@ -30,6 +31,7 @@ impl StepDefinition {
retry_policy: None,
timeout: None,
on_failure: FailureAction::FailWorkflow,
depends_on: Vec::new(),
}
}
@@ -47,6 +49,14 @@ impl StepDefinition {
self.on_failure = action;
self
}
/// Set dependencies for this step.
/// The step will only run after all specified dependencies have completed successfully.
/// Empty slice means no dependencies - step can run immediately in parallel with others.
pub fn depends_on(mut self, deps: &[&str]) -> Self {
self.depends_on = deps.iter().map(|s| StepId::new(*s)).collect();
self
}
}
/// Complete workflow definition
@@ -95,4 +105,82 @@ impl WorkflowDefinition {
pub fn get_timeout(&self, step: &StepDefinition) -> Duration {
step.timeout.unwrap_or(self.default_timeout)
}
/// Validate the workflow DAG structure.
/// Returns an error if:
/// - A step depends on a non-existent step
/// - There's a cycle in the dependencies
pub fn validate(&self) -> Result<(), String> {
let step_ids: HashSet<_> = self.steps.iter().map(|s| &s.id).collect();
// Check all dependencies exist
for step in &self.steps {
for dep in &step.depends_on {
if !step_ids.contains(dep) {
return Err(format!(
"Step '{}' depends on non-existent step '{}'",
step.id, dep
));
}
}
}
// Check for cycles using DFS
let mut visited = HashSet::new();
let mut rec_stack = HashSet::new();
for step in &self.steps {
if self.has_cycle(&step.id, &mut visited, &mut rec_stack) {
return Err(format!("Cycle detected involving step '{}'", step.id));
}
}
Ok(())
}
/// DFS helper for cycle detection
fn has_cycle(
&self,
step_id: &StepId,
visited: &mut HashSet<StepId>,
rec_stack: &mut HashSet<StepId>,
) -> bool {
if rec_stack.contains(step_id) {
return true; // Back edge found - cycle!
}
if visited.contains(step_id) {
return false; // Already fully processed
}
visited.insert(step_id.clone());
rec_stack.insert(step_id.clone());
// Find the step and check its dependencies
if let Some(step) = self.steps.iter().find(|s| &s.id == step_id) {
for dep in &step.depends_on {
if self.has_cycle(dep, visited, rec_stack) {
return true;
}
}
}
rec_stack.remove(step_id);
false
}
/// Get steps that have no dependencies (can run immediately)
pub fn get_initial_steps(&self) -> Vec<&StepDefinition> {
self.steps
.iter()
.filter(|s| s.depends_on.is_empty())
.collect()
}
/// Get steps that depend on the given step
pub fn get_dependents(&self, step_id: &StepId) -> Vec<&StepDefinition> {
self.steps
.iter()
.filter(|s| s.depends_on.contains(step_id))
.collect()
}
}
+214 -70
View File
@@ -1,11 +1,19 @@
//! Workflow execution engine
//!
//! Supports DAG-based parallel execution of workflow steps.
//! Steps with no dependencies run in parallel, steps with dependencies
//! wait for all dependencies to complete successfully.
use std::{collections::HashMap, sync::Arc, time::Duration};
use std::{
collections::{HashMap, HashSet},
sync::Arc,
time::Duration,
};
use backoff::{backoff::Backoff, ExponentialBackoffBuilder};
use chrono::Utc;
use parking_lot::RwLock;
use tokio::time::timeout;
use tokio::{sync::mpsc, time::timeout};
use super::{
definition::{StepDefinition, WorkflowDefinition},
@@ -14,6 +22,39 @@ use super::{
types::*,
};
/// Consolidated step execution tracking to minimize lock contention.
/// Single lock instead of 4 separate locks.
#[derive(Default)]
struct StepTracker {
completed: HashSet<StepId>,
failed: HashSet<StepId>,
skipped: HashSet<StepId>,
running: HashSet<StepId>,
}
impl StepTracker {
fn total_processed(&self) -> usize {
self.completed.len() + self.failed.len() + self.skipped.len()
}
fn is_step_processable(&self, step_id: &StepId) -> bool {
!self.completed.contains(step_id)
&& !self.failed.contains(step_id)
&& !self.skipped.contains(step_id)
&& !self.running.contains(step_id)
}
fn are_dependencies_satisfied(&self, depends_on: &[StepId]) -> bool {
depends_on
.iter()
.all(|dep| self.completed.contains(dep) || self.skipped.contains(dep))
}
fn has_failed_dependency(&self, depends_on: &[StepId]) -> bool {
depends_on.iter().any(|dep| self.failed.contains(dep))
}
}
/// Linear backoff implementation that increases delay by a fixed amount each retry
struct LinearBackoff {
current: Duration,
@@ -32,15 +73,15 @@ impl LinearBackoff {
}
impl Backoff for LinearBackoff {
fn reset(&mut self) {
self.current = self.increment;
}
fn next_backoff(&mut self) -> Option<Duration> {
let next = self.current;
self.current = (self.current + self.increment).min(self.max);
Some(next)
}
fn reset(&mut self) {
self.current = self.increment;
}
}
/// Main workflow execution engine
@@ -93,9 +134,13 @@ impl WorkflowEngine {
}
/// Register a workflow definition
pub fn register_workflow(&self, definition: WorkflowDefinition) {
pub fn register_workflow(&self, definition: WorkflowDefinition) -> Result<(), String> {
// Validate DAG once at registration, not on every execution
definition.validate()?;
let id = definition.id.clone();
self.definitions.write().insert(id, Arc::new(definition));
Ok(())
}
/// Get the event bus for subscribing to workflow events
@@ -159,15 +204,23 @@ impl WorkflowEngine {
Ok(instance_id)
}
/// Execute a workflow (internal)
/// Execute a workflow with DAG-based parallel execution
async fn execute_workflow(
&self,
instance_id: WorkflowInstanceId,
definition: Arc<WorkflowDefinition>,
) -> WorkflowResult<()> {
let start_time = std::time::Instant::now();
let step_count = definition.steps.len();
for step in &definition.steps {
// Consolidated tracking state - single lock instead of 4
let tracker: Arc<RwLock<StepTracker>> = Arc::new(RwLock::new(StepTracker::default()));
// Channel for step completion notifications
// Capacity equals step count to prevent blocking on send
let (tx, mut rx) = mpsc::channel::<(StepId, StepResult)>(step_count.max(1));
loop {
// Check if workflow was cancelled
let state = self.state_store.load(instance_id)?;
if state.status == WorkflowStatus::Cancelled {
@@ -177,67 +230,161 @@ impl WorkflowEngine {
return Ok(());
}
// Execute step with retry
match self
.execute_step_with_retry(instance_id, step, &definition)
.await
{
Ok(StepResult::Success) => {
// Continue to next step
}
Ok(StepResult::Skip) => {
// Step was skipped, continue to next
continue;
}
Ok(StepResult::Failure) | Err(_) => {
// Handle failure based on failure action
match step.on_failure {
FailureAction::FailWorkflow => {
let error_msg = format!("Step {} failed", step.id);
self.state_store.update(instance_id, |s| {
s.status = WorkflowStatus::Failed;
})?;
// Single lock acquisition for all tracking state reads
let (ready_step_indices, total_processed, running_count, blocked_by_failure) = {
let t = tracker.read();
self.event_bus
.publish(WorkflowEvent::WorkflowFailed {
instance_id,
failed_step: step.id.clone(),
error: error_msg,
})
.await;
let ready: Vec<usize> = definition
.steps
.iter()
.enumerate()
.filter(|(_, step)| {
t.is_step_processable(&step.id)
&& t.are_dependencies_satisfied(&step.depends_on)
&& !t.has_failed_dependency(&step.depends_on)
})
.map(|(i, _)| i)
.collect();
return Ok(());
}
FailureAction::ContinueNextStep => {
// Mark step as skipped and continue
self.state_store.update(instance_id, |s| {
if let Some(step_state) = s.step_states.get_mut(&step.id) {
step_state.status = StepStatus::Skipped;
let processed = t.total_processed();
let running = t.running.len();
// Check for blocked steps only if needed
let blocked = ready.is_empty()
&& running == 0
&& definition.steps.iter().any(|step| {
t.is_step_processable(&step.id) && t.has_failed_dependency(&step.depends_on)
});
(ready, processed, running, blocked)
};
// Check if we're done
if total_processed == step_count {
break;
}
// Handle blocked workflow
if ready_step_indices.is_empty() && running_count == 0 {
if blocked_by_failure {
self.state_store.update(instance_id, |s| {
s.status = WorkflowStatus::Failed;
})?;
let failed_step = tracker.read().failed.iter().next().cloned();
self.event_bus
.publish(WorkflowEvent::WorkflowFailed {
instance_id,
failed_step: failed_step.unwrap_or_else(|| StepId::new("unknown")),
error: "Workflow failed due to step dependency failure".to_string(),
})
.await;
return Ok(());
}
break;
}
// Launch ready steps in parallel
for step_idx in ready_step_indices {
let step = &definition.steps[step_idx];
tracker.write().running.insert(step.id.clone());
let engine = self.clone_for_execution();
let def = Arc::clone(&definition);
let step_id = step.id.clone();
let tx = tx.clone();
let tracker = Arc::clone(&tracker);
tokio::spawn(async move {
let step = &def.steps[step_idx];
let result = engine
.execute_step_with_retry(instance_id, step, &def)
.await;
// Single lock acquisition to update tracking state
{
let mut t = tracker.write();
t.running.remove(&step_id);
match result {
Ok(StepResult::Success) => {
t.completed.insert(step_id.clone());
}
Ok(StepResult::Skip) => {
t.skipped.insert(step_id.clone());
}
Ok(StepResult::Failure) | Err(_) => match step.on_failure {
FailureAction::FailWorkflow | FailureAction::RetryIndefinitely => {
t.failed.insert(step_id.clone());
}
})?;
continue;
}
FailureAction::RetryIndefinitely => {
// This should not happen as execute_step_with_retry handles it
unreachable!("RetryIndefinitely should be handled in retry logic");
FailureAction::ContinueNextStep => {
if let Err(e) = engine.state_store.update(instance_id, |s| {
if let Some(step_state) = s.step_states.get_mut(&step_id) {
step_state.status = StepStatus::Skipped;
}
}) {
tracing::warn!(
step_id = %step_id,
error = ?e,
"Failed to update step state to Skipped"
);
}
t.skipped.insert(step_id.clone());
}
},
}
}
// Send completion signal (result determines the message)
let signal = match result {
Ok(r) => r,
Err(_) => StepResult::Failure,
};
let _ = tx.send((step_id, signal)).await;
});
}
// Wait for at least one step to complete (if any running)
if !tracker.read().running.is_empty() {
if let Some((completed_step_id, result)) = rx.recv().await {
tracing::debug!(
step_id = %completed_step_id,
result = ?result,
"Step completed"
);
}
}
}
// Workflow completed successfully
self.state_store.update(instance_id, |s| {
s.status = WorkflowStatus::Completed;
})?;
// Check final status - single lock read
let has_failures = !tracker.read().failed.is_empty();
let duration = start_time.elapsed();
self.event_bus
.publish(WorkflowEvent::WorkflowCompleted {
instance_id,
duration,
})
.await;
if has_failures {
self.state_store.update(instance_id, |s| {
s.status = WorkflowStatus::Failed;
})?;
let failed_step = tracker.read().failed.iter().next().cloned();
self.event_bus
.publish(WorkflowEvent::WorkflowFailed {
instance_id,
failed_step: failed_step.unwrap_or_else(|| StepId::new("unknown")),
error: "One or more steps failed".to_string(),
})
.await;
} else {
self.state_store.update(instance_id, |s| {
s.status = WorkflowStatus::Completed;
})?;
let duration = start_time.elapsed();
self.event_bus
.publish(WorkflowEvent::WorkflowCompleted {
instance_id,
duration,
})
.await;
}
Ok(())
}
@@ -262,12 +409,9 @@ impl WorkflowEngine {
let mut backoff = Self::create_backoff(&retry_policy.backoff);
loop {
// Check for cancellation before starting/retrying step
{
let state = self.state_store.load(instance_id)?;
if state.status == WorkflowStatus::Cancelled {
return Err(WorkflowError::Cancelled(instance_id));
}
// Check for cancellation before starting/retrying step (optimized: no full state load)
if self.state_store.is_cancelled(instance_id)? {
return Err(WorkflowError::Cancelled(instance_id));
}
// Update step state
@@ -293,8 +437,8 @@ impl WorkflowEngine {
})
.await;
// Get current context
let mut context = self.state_store.load(instance_id)?.context;
// Get current context (optimized: only clone context, not full state)
let mut context = self.state_store.get_context(instance_id)?;
// Execute step with timeout
let step_start = std::time::Instant::now();
@@ -302,9 +446,9 @@ impl WorkflowEngine {
let step_duration = step_start.elapsed();
// Save updated context
// Save updated context (use std::mem::replace to avoid extra clone)
self.state_store.update(instance_id, |s| {
s.context = context.clone();
s.context = std::mem::replace(&mut context, WorkflowContext::new(instance_id));
})?;
match result {
+22
View File
@@ -86,6 +86,28 @@ impl WorkflowStateStore {
Ok(())
}
/// Get just the workflow context without cloning the entire state.
/// More efficient when you only need the context for step execution.
pub fn get_context(
&self,
instance_id: WorkflowInstanceId,
) -> WorkflowResult<super::types::WorkflowContext> {
self.states
.read()
.get(&instance_id)
.map(|s| s.context.clone())
.ok_or(WorkflowError::NotFound(instance_id))
}
/// Check if workflow is cancelled without loading full state
pub fn is_cancelled(&self, instance_id: WorkflowInstanceId) -> WorkflowResult<bool> {
self.states
.read()
.get(&instance_id)
.map(|s| s.status == WorkflowStatus::Cancelled)
.ok_or(WorkflowError::NotFound(instance_id))
}
/// Get count of workflows by status
pub fn count_by_status(&self, status: WorkflowStatus) -> usize {
self.states