[model-gateway] feat: add DAG parallel execution support and workflow optimization (#14999)
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
//! Workflow definition types
|
||||
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use std::{collections::HashSet, sync::Arc, time::Duration};
|
||||
|
||||
use super::{
|
||||
executor::StepExecutor,
|
||||
@@ -15,6 +15,7 @@ pub struct StepDefinition {
|
||||
pub retry_policy: Option<RetryPolicy>,
|
||||
pub timeout: Option<Duration>,
|
||||
pub on_failure: FailureAction,
|
||||
pub depends_on: Vec<StepId>,
|
||||
}
|
||||
|
||||
impl StepDefinition {
|
||||
@@ -30,6 +31,7 @@ impl StepDefinition {
|
||||
retry_policy: None,
|
||||
timeout: None,
|
||||
on_failure: FailureAction::FailWorkflow,
|
||||
depends_on: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,6 +49,14 @@ impl StepDefinition {
|
||||
self.on_failure = action;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set dependencies for this step.
|
||||
/// The step will only run after all specified dependencies have completed successfully.
|
||||
/// Empty slice means no dependencies - step can run immediately in parallel with others.
|
||||
pub fn depends_on(mut self, deps: &[&str]) -> Self {
|
||||
self.depends_on = deps.iter().map(|s| StepId::new(*s)).collect();
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Complete workflow definition
|
||||
@@ -95,4 +105,82 @@ impl WorkflowDefinition {
|
||||
pub fn get_timeout(&self, step: &StepDefinition) -> Duration {
|
||||
step.timeout.unwrap_or(self.default_timeout)
|
||||
}
|
||||
|
||||
/// Validate the workflow DAG structure.
|
||||
/// Returns an error if:
|
||||
/// - A step depends on a non-existent step
|
||||
/// - There's a cycle in the dependencies
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
let step_ids: HashSet<_> = self.steps.iter().map(|s| &s.id).collect();
|
||||
|
||||
// Check all dependencies exist
|
||||
for step in &self.steps {
|
||||
for dep in &step.depends_on {
|
||||
if !step_ids.contains(dep) {
|
||||
return Err(format!(
|
||||
"Step '{}' depends on non-existent step '{}'",
|
||||
step.id, dep
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for cycles using DFS
|
||||
let mut visited = HashSet::new();
|
||||
let mut rec_stack = HashSet::new();
|
||||
|
||||
for step in &self.steps {
|
||||
if self.has_cycle(&step.id, &mut visited, &mut rec_stack) {
|
||||
return Err(format!("Cycle detected involving step '{}'", step.id));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// DFS helper for cycle detection
|
||||
fn has_cycle(
|
||||
&self,
|
||||
step_id: &StepId,
|
||||
visited: &mut HashSet<StepId>,
|
||||
rec_stack: &mut HashSet<StepId>,
|
||||
) -> bool {
|
||||
if rec_stack.contains(step_id) {
|
||||
return true; // Back edge found - cycle!
|
||||
}
|
||||
if visited.contains(step_id) {
|
||||
return false; // Already fully processed
|
||||
}
|
||||
|
||||
visited.insert(step_id.clone());
|
||||
rec_stack.insert(step_id.clone());
|
||||
|
||||
// Find the step and check its dependencies
|
||||
if let Some(step) = self.steps.iter().find(|s| &s.id == step_id) {
|
||||
for dep in &step.depends_on {
|
||||
if self.has_cycle(dep, visited, rec_stack) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rec_stack.remove(step_id);
|
||||
false
|
||||
}
|
||||
|
||||
/// Get steps that have no dependencies (can run immediately)
|
||||
pub fn get_initial_steps(&self) -> Vec<&StepDefinition> {
|
||||
self.steps
|
||||
.iter()
|
||||
.filter(|s| s.depends_on.is_empty())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Get steps that depend on the given step
|
||||
pub fn get_dependents(&self, step_id: &StepId) -> Vec<&StepDefinition> {
|
||||
self.steps
|
||||
.iter()
|
||||
.filter(|s| s.depends_on.contains(step_id))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,19 @@
|
||||
//! Workflow execution engine
|
||||
//!
|
||||
//! Supports DAG-based parallel execution of workflow steps.
|
||||
//! Steps with no dependencies run in parallel, steps with dependencies
|
||||
//! wait for all dependencies to complete successfully.
|
||||
|
||||
use std::{collections::HashMap, sync::Arc, time::Duration};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use backoff::{backoff::Backoff, ExponentialBackoffBuilder};
|
||||
use chrono::Utc;
|
||||
use parking_lot::RwLock;
|
||||
use tokio::time::timeout;
|
||||
use tokio::{sync::mpsc, time::timeout};
|
||||
|
||||
use super::{
|
||||
definition::{StepDefinition, WorkflowDefinition},
|
||||
@@ -14,6 +22,39 @@ use super::{
|
||||
types::*,
|
||||
};
|
||||
|
||||
/// Consolidated step execution tracking to minimize lock contention.
|
||||
/// Single lock instead of 4 separate locks.
|
||||
#[derive(Default)]
|
||||
struct StepTracker {
|
||||
completed: HashSet<StepId>,
|
||||
failed: HashSet<StepId>,
|
||||
skipped: HashSet<StepId>,
|
||||
running: HashSet<StepId>,
|
||||
}
|
||||
|
||||
impl StepTracker {
|
||||
fn total_processed(&self) -> usize {
|
||||
self.completed.len() + self.failed.len() + self.skipped.len()
|
||||
}
|
||||
|
||||
fn is_step_processable(&self, step_id: &StepId) -> bool {
|
||||
!self.completed.contains(step_id)
|
||||
&& !self.failed.contains(step_id)
|
||||
&& !self.skipped.contains(step_id)
|
||||
&& !self.running.contains(step_id)
|
||||
}
|
||||
|
||||
fn are_dependencies_satisfied(&self, depends_on: &[StepId]) -> bool {
|
||||
depends_on
|
||||
.iter()
|
||||
.all(|dep| self.completed.contains(dep) || self.skipped.contains(dep))
|
||||
}
|
||||
|
||||
fn has_failed_dependency(&self, depends_on: &[StepId]) -> bool {
|
||||
depends_on.iter().any(|dep| self.failed.contains(dep))
|
||||
}
|
||||
}
|
||||
|
||||
/// Linear backoff implementation that increases delay by a fixed amount each retry
|
||||
struct LinearBackoff {
|
||||
current: Duration,
|
||||
@@ -32,15 +73,15 @@ impl LinearBackoff {
|
||||
}
|
||||
|
||||
impl Backoff for LinearBackoff {
|
||||
fn reset(&mut self) {
|
||||
self.current = self.increment;
|
||||
}
|
||||
|
||||
fn next_backoff(&mut self) -> Option<Duration> {
|
||||
let next = self.current;
|
||||
self.current = (self.current + self.increment).min(self.max);
|
||||
Some(next)
|
||||
}
|
||||
|
||||
fn reset(&mut self) {
|
||||
self.current = self.increment;
|
||||
}
|
||||
}
|
||||
|
||||
/// Main workflow execution engine
|
||||
@@ -93,9 +134,13 @@ impl WorkflowEngine {
|
||||
}
|
||||
|
||||
/// Register a workflow definition
|
||||
pub fn register_workflow(&self, definition: WorkflowDefinition) {
|
||||
pub fn register_workflow(&self, definition: WorkflowDefinition) -> Result<(), String> {
|
||||
// Validate DAG once at registration, not on every execution
|
||||
definition.validate()?;
|
||||
|
||||
let id = definition.id.clone();
|
||||
self.definitions.write().insert(id, Arc::new(definition));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the event bus for subscribing to workflow events
|
||||
@@ -159,15 +204,23 @@ impl WorkflowEngine {
|
||||
Ok(instance_id)
|
||||
}
|
||||
|
||||
/// Execute a workflow (internal)
|
||||
/// Execute a workflow with DAG-based parallel execution
|
||||
async fn execute_workflow(
|
||||
&self,
|
||||
instance_id: WorkflowInstanceId,
|
||||
definition: Arc<WorkflowDefinition>,
|
||||
) -> WorkflowResult<()> {
|
||||
let start_time = std::time::Instant::now();
|
||||
let step_count = definition.steps.len();
|
||||
|
||||
for step in &definition.steps {
|
||||
// Consolidated tracking state - single lock instead of 4
|
||||
let tracker: Arc<RwLock<StepTracker>> = Arc::new(RwLock::new(StepTracker::default()));
|
||||
|
||||
// Channel for step completion notifications
|
||||
// Capacity equals step count to prevent blocking on send
|
||||
let (tx, mut rx) = mpsc::channel::<(StepId, StepResult)>(step_count.max(1));
|
||||
|
||||
loop {
|
||||
// Check if workflow was cancelled
|
||||
let state = self.state_store.load(instance_id)?;
|
||||
if state.status == WorkflowStatus::Cancelled {
|
||||
@@ -177,67 +230,161 @@ impl WorkflowEngine {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Execute step with retry
|
||||
match self
|
||||
.execute_step_with_retry(instance_id, step, &definition)
|
||||
.await
|
||||
{
|
||||
Ok(StepResult::Success) => {
|
||||
// Continue to next step
|
||||
}
|
||||
Ok(StepResult::Skip) => {
|
||||
// Step was skipped, continue to next
|
||||
continue;
|
||||
}
|
||||
Ok(StepResult::Failure) | Err(_) => {
|
||||
// Handle failure based on failure action
|
||||
match step.on_failure {
|
||||
FailureAction::FailWorkflow => {
|
||||
let error_msg = format!("Step {} failed", step.id);
|
||||
self.state_store.update(instance_id, |s| {
|
||||
s.status = WorkflowStatus::Failed;
|
||||
})?;
|
||||
// Single lock acquisition for all tracking state reads
|
||||
let (ready_step_indices, total_processed, running_count, blocked_by_failure) = {
|
||||
let t = tracker.read();
|
||||
|
||||
self.event_bus
|
||||
.publish(WorkflowEvent::WorkflowFailed {
|
||||
instance_id,
|
||||
failed_step: step.id.clone(),
|
||||
error: error_msg,
|
||||
})
|
||||
.await;
|
||||
let ready: Vec<usize> = definition
|
||||
.steps
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, step)| {
|
||||
t.is_step_processable(&step.id)
|
||||
&& t.are_dependencies_satisfied(&step.depends_on)
|
||||
&& !t.has_failed_dependency(&step.depends_on)
|
||||
})
|
||||
.map(|(i, _)| i)
|
||||
.collect();
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
FailureAction::ContinueNextStep => {
|
||||
// Mark step as skipped and continue
|
||||
self.state_store.update(instance_id, |s| {
|
||||
if let Some(step_state) = s.step_states.get_mut(&step.id) {
|
||||
step_state.status = StepStatus::Skipped;
|
||||
let processed = t.total_processed();
|
||||
let running = t.running.len();
|
||||
|
||||
// Check for blocked steps only if needed
|
||||
let blocked = ready.is_empty()
|
||||
&& running == 0
|
||||
&& definition.steps.iter().any(|step| {
|
||||
t.is_step_processable(&step.id) && t.has_failed_dependency(&step.depends_on)
|
||||
});
|
||||
|
||||
(ready, processed, running, blocked)
|
||||
};
|
||||
|
||||
// Check if we're done
|
||||
if total_processed == step_count {
|
||||
break;
|
||||
}
|
||||
|
||||
// Handle blocked workflow
|
||||
if ready_step_indices.is_empty() && running_count == 0 {
|
||||
if blocked_by_failure {
|
||||
self.state_store.update(instance_id, |s| {
|
||||
s.status = WorkflowStatus::Failed;
|
||||
})?;
|
||||
|
||||
let failed_step = tracker.read().failed.iter().next().cloned();
|
||||
self.event_bus
|
||||
.publish(WorkflowEvent::WorkflowFailed {
|
||||
instance_id,
|
||||
failed_step: failed_step.unwrap_or_else(|| StepId::new("unknown")),
|
||||
error: "Workflow failed due to step dependency failure".to_string(),
|
||||
})
|
||||
.await;
|
||||
return Ok(());
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// Launch ready steps in parallel
|
||||
for step_idx in ready_step_indices {
|
||||
let step = &definition.steps[step_idx];
|
||||
tracker.write().running.insert(step.id.clone());
|
||||
|
||||
let engine = self.clone_for_execution();
|
||||
let def = Arc::clone(&definition);
|
||||
let step_id = step.id.clone();
|
||||
let tx = tx.clone();
|
||||
let tracker = Arc::clone(&tracker);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let step = &def.steps[step_idx];
|
||||
let result = engine
|
||||
.execute_step_with_retry(instance_id, step, &def)
|
||||
.await;
|
||||
|
||||
// Single lock acquisition to update tracking state
|
||||
{
|
||||
let mut t = tracker.write();
|
||||
t.running.remove(&step_id);
|
||||
|
||||
match result {
|
||||
Ok(StepResult::Success) => {
|
||||
t.completed.insert(step_id.clone());
|
||||
}
|
||||
Ok(StepResult::Skip) => {
|
||||
t.skipped.insert(step_id.clone());
|
||||
}
|
||||
Ok(StepResult::Failure) | Err(_) => match step.on_failure {
|
||||
FailureAction::FailWorkflow | FailureAction::RetryIndefinitely => {
|
||||
t.failed.insert(step_id.clone());
|
||||
}
|
||||
})?;
|
||||
continue;
|
||||
}
|
||||
FailureAction::RetryIndefinitely => {
|
||||
// This should not happen as execute_step_with_retry handles it
|
||||
unreachable!("RetryIndefinitely should be handled in retry logic");
|
||||
FailureAction::ContinueNextStep => {
|
||||
if let Err(e) = engine.state_store.update(instance_id, |s| {
|
||||
if let Some(step_state) = s.step_states.get_mut(&step_id) {
|
||||
step_state.status = StepStatus::Skipped;
|
||||
}
|
||||
}) {
|
||||
tracing::warn!(
|
||||
step_id = %step_id,
|
||||
error = ?e,
|
||||
"Failed to update step state to Skipped"
|
||||
);
|
||||
}
|
||||
t.skipped.insert(step_id.clone());
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Send completion signal (result determines the message)
|
||||
let signal = match result {
|
||||
Ok(r) => r,
|
||||
Err(_) => StepResult::Failure,
|
||||
};
|
||||
let _ = tx.send((step_id, signal)).await;
|
||||
});
|
||||
}
|
||||
|
||||
// Wait for at least one step to complete (if any running)
|
||||
if !tracker.read().running.is_empty() {
|
||||
if let Some((completed_step_id, result)) = rx.recv().await {
|
||||
tracing::debug!(
|
||||
step_id = %completed_step_id,
|
||||
result = ?result,
|
||||
"Step completed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Workflow completed successfully
|
||||
self.state_store.update(instance_id, |s| {
|
||||
s.status = WorkflowStatus::Completed;
|
||||
})?;
|
||||
// Check final status - single lock read
|
||||
let has_failures = !tracker.read().failed.is_empty();
|
||||
|
||||
let duration = start_time.elapsed();
|
||||
self.event_bus
|
||||
.publish(WorkflowEvent::WorkflowCompleted {
|
||||
instance_id,
|
||||
duration,
|
||||
})
|
||||
.await;
|
||||
if has_failures {
|
||||
self.state_store.update(instance_id, |s| {
|
||||
s.status = WorkflowStatus::Failed;
|
||||
})?;
|
||||
|
||||
let failed_step = tracker.read().failed.iter().next().cloned();
|
||||
self.event_bus
|
||||
.publish(WorkflowEvent::WorkflowFailed {
|
||||
instance_id,
|
||||
failed_step: failed_step.unwrap_or_else(|| StepId::new("unknown")),
|
||||
error: "One or more steps failed".to_string(),
|
||||
})
|
||||
.await;
|
||||
} else {
|
||||
self.state_store.update(instance_id, |s| {
|
||||
s.status = WorkflowStatus::Completed;
|
||||
})?;
|
||||
|
||||
let duration = start_time.elapsed();
|
||||
self.event_bus
|
||||
.publish(WorkflowEvent::WorkflowCompleted {
|
||||
instance_id,
|
||||
duration,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -262,12 +409,9 @@ impl WorkflowEngine {
|
||||
let mut backoff = Self::create_backoff(&retry_policy.backoff);
|
||||
|
||||
loop {
|
||||
// Check for cancellation before starting/retrying step
|
||||
{
|
||||
let state = self.state_store.load(instance_id)?;
|
||||
if state.status == WorkflowStatus::Cancelled {
|
||||
return Err(WorkflowError::Cancelled(instance_id));
|
||||
}
|
||||
// Check for cancellation before starting/retrying step (optimized: no full state load)
|
||||
if self.state_store.is_cancelled(instance_id)? {
|
||||
return Err(WorkflowError::Cancelled(instance_id));
|
||||
}
|
||||
|
||||
// Update step state
|
||||
@@ -293,8 +437,8 @@ impl WorkflowEngine {
|
||||
})
|
||||
.await;
|
||||
|
||||
// Get current context
|
||||
let mut context = self.state_store.load(instance_id)?.context;
|
||||
// Get current context (optimized: only clone context, not full state)
|
||||
let mut context = self.state_store.get_context(instance_id)?;
|
||||
|
||||
// Execute step with timeout
|
||||
let step_start = std::time::Instant::now();
|
||||
@@ -302,9 +446,9 @@ impl WorkflowEngine {
|
||||
|
||||
let step_duration = step_start.elapsed();
|
||||
|
||||
// Save updated context
|
||||
// Save updated context (use std::mem::replace to avoid extra clone)
|
||||
self.state_store.update(instance_id, |s| {
|
||||
s.context = context.clone();
|
||||
s.context = std::mem::replace(&mut context, WorkflowContext::new(instance_id));
|
||||
})?;
|
||||
|
||||
match result {
|
||||
|
||||
@@ -86,6 +86,28 @@ impl WorkflowStateStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get just the workflow context without cloning the entire state.
|
||||
/// More efficient when you only need the context for step execution.
|
||||
pub fn get_context(
|
||||
&self,
|
||||
instance_id: WorkflowInstanceId,
|
||||
) -> WorkflowResult<super::types::WorkflowContext> {
|
||||
self.states
|
||||
.read()
|
||||
.get(&instance_id)
|
||||
.map(|s| s.context.clone())
|
||||
.ok_or(WorkflowError::NotFound(instance_id))
|
||||
}
|
||||
|
||||
/// Check if workflow is cancelled without loading full state
|
||||
pub fn is_cancelled(&self, instance_id: WorkflowInstanceId) -> WorkflowResult<bool> {
|
||||
self.states
|
||||
.read()
|
||||
.get(&instance_id)
|
||||
.map(|s| s.status == WorkflowStatus::Cancelled)
|
||||
.ok_or(WorkflowError::NotFound(instance_id))
|
||||
}
|
||||
|
||||
/// Get count of workflows by status
|
||||
pub fn count_by_status(&self, status: WorkflowStatus) -> usize {
|
||||
self.states
|
||||
|
||||
Reference in New Issue
Block a user