chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[diffusion] model: support Hunyuan3D-2 (#18170)

Browse Source 
Co-authored-by: yingluosanqian <yingluosanqian@gmail.com>
Co-authored-by: daiweitao <dwti614707404@163.com>
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
Prozac614
2026-03-02 12:28:05 +08:00
committed by GitHub
co-authored by yingluosanqian daiweitao Mick
parent f6ee6dc8c3
commit 57c5c343d7
43 changed files with 8082 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/multimodal_gen/test/server/perf_baselines.json
+66
View File
@@ -1920,6 +1920,72 @@
"expected_avg_denoise_ms": 173.83,
"expected_median_denoise_ms": 178.08
},
"hunyuan3d_shape_gen": {
"stages_ms": {
"Hunyuan3DShapeBeforeDenoisingStage": 31.42,
"Hunyuan3DShapeDenoisingStage": 3259.83,
"Hunyuan3DShapeExportStage": 8735.55,
"Hunyuan3DShapeSaveStage": 981.64,
"Hunyuan3DPaintPreprocessStage": 226071.67,
"Hunyuan3DPaintTexGenStage": 11083.05,
"Hunyuan3DPaintPostprocessStage": 7469.29
},
"denoise_step_ms": {
"0": 32.26,
"1": 63.34,
"2": 65.44,
"3": 65.44,
"4": 65.6,
"5": 65.81,
"6": 65.82,
"7": 65.48,
"8": 65.9,
"9": 65.77,
"10": 65.54,
"11": 65.68,
"12": 65.85,
"13": 65.77,
"14": 65.7,
"15": 65.78,
"16": 66.0,
"17": 66.15,
"18": 65.91,
"19": 66.5,
"20": 65.76,
"21": 66.08,
"22": 66.06,
"23": 66.23,
"24": 65.79,
"25": 65.58,
"26": 65.88,
"27": 65.67,
"28": 65.87,
"29": 66.09,
"30": 65.81,
"31": 65.91,
"32": 66.18,
"33": 65.93,
"34": 66.26,
"35": 66.26,
"36": 66.27,
"37": 65.57,
"38": 66.02,
"39": 66.19,
"40": 65.23,
"41": 66.11,
"42": 66.18,
"43": 65.86,
"44": 65.86,
"45": 65.92,
"46": 65.65,
"47": 65.78,
"48": 66.01,
"49": 66.08
},
"expected_e2e_ms": 257696.97,
"expected_avg_denoise_ms": 65.16,
"expected_median_denoise_ms": 65.86
},
"wan2_1_t2v_1.3b_frame_interp_2x": {
"stages_ms": {
"InputValidationStage": 0.03,
python/sglang/multimodal_gen/test/server/test_server_common.py
+12
View File
@@ -734,6 +734,7 @@ Consider updating perf_baselines.json with the snippets below:
modality_to_valid_task_types = {
"image": {"T2I", "I2I", "TI2I"},
"video": {"T2V", "I2V", "TI2V"},
"3d": {"I2M"},
}
valid_task_types = modality_to_valid_task_types.get(
case.server_args.modality, set()
@@ -858,6 +859,17 @@ Consider updating perf_baselines.json with the snippets below:
# Validation 1: Performance
self._validate_and_record(case, perf_record)
# Mesh correctness check (Chamfer Distance) for 3D models
if case.server_args.custom_validator == "mesh":
from sglang.multimodal_gen.test.server.test_server_utils import (
MESH_OUTPUT_PATHS,
validate_mesh_correctness,
)
mesh_path = MESH_OUTPUT_PATHS.pop(case.id, None)
if mesh_path:
validate_mesh_correctness(mesh_path)
# Test /v1/models endpoint for router compatibility
self._test_v1_models_endpoint(diffusion_server, case)
self._test_t2v_rejects_input_reference(diffusion_server, case)
python/sglang/multimodal_gen/test/server/test_server_utils.py
+190 -1
View File
@@ -50,6 +50,10 @@ logger = init_logger(__name__)
globally_suppress_loggers()
# Tracks mesh output file paths from generate_mesh for later correctness validation.
# Keyed by case_id, cleaned up after use.
MESH_OUTPUT_PATHS: dict[str, str] = {}
def download_image_from_url(url: str) -> Path:
"""Download an image from a URL to a temporary file.
@@ -637,10 +641,99 @@ class VideoPerformanceValidator(PerformanceValidator):
)
class MeshValidator(PerformanceValidator):
"""Validator for 3D mesh generation. Inherits perf validation from PerformanceValidator."""
pass
HUNYUAN3D_REFERENCE_URL = (
"https://raw.githubusercontent.com/sgl-project/sgl-test-files/"
"main/diffusion-ci/consistency_gt/1-gpu/hunyuan3d_2_0/hunyuan3d.glb"
)
def _download_reference_mesh(url: str) -> Path:
"""Download a reference mesh from URL, caching in temp dir."""
import hashlib
cache_name = f"ref_mesh_{hashlib.md5(url.encode()).hexdigest()}.glb"
cache_path = Path(tempfile.gettempdir()) / cache_name
if cache_path.exists():
logger.info(f"Using cached reference mesh: {cache_path}")
return cache_path
logger.info(f"Downloading reference mesh from: {url}")
with urlopen(url, timeout=60) as resp:
cache_path.write_bytes(resp.read())
logger.info(f"Reference mesh cached at: {cache_path}")
return cache_path
def validate_mesh_correctness(
generated_mesh_path: str,
reference_url: str = HUNYUAN3D_REFERENCE_URL,
num_sample_points: int = 4096,
cd_threshold_ratio: float = 0.01,
random_seed: int = 42,
):
"""Validate mesh geometric similarity against a reference via Chamfer Distance.
Downloads the reference mesh from a URL (cached), samples point clouds from
both meshes, and asserts Chamfer Distance is within threshold.
"""
import numpy as np
try:
import trimesh
except ImportError:
pytest.fail("trimesh is required for mesh validation: pip install trimesh")
from scipy.spatial import cKDTree
# Load generated mesh
generated_mesh = trimesh.load(generated_mesh_path)
if isinstance(generated_mesh, trimesh.Scene):
generated_mesh = generated_mesh.dump(concatenate=True)
# Download and load reference mesh
ref_path = _download_reference_mesh(reference_url)
reference_mesh = trimesh.load(str(ref_path))
if isinstance(reference_mesh, trimesh.Scene):
reference_mesh = reference_mesh.dump(concatenate=True)
# Bounding box diagonal for threshold normalization
ref_bbox = reference_mesh.bounding_box.bounds
bbox_diagonal = float(np.linalg.norm(ref_bbox[1] - ref_bbox[0]))
cd_threshold = cd_threshold_ratio * bbox_diagonal
# Sample point clouds
np.random.seed(random_seed)
gen_points = np.array(
generated_mesh.sample(num_sample_points, return_index=True)[0]
)
ref_points = np.array(
reference_mesh.sample(num_sample_points, return_index=True)[0]
)
# Bidirectional Chamfer Distance
tree1 = cKDTree(gen_points)
tree2 = cKDTree(ref_points)
forward_cd = float(np.mean(tree2.query(gen_points)[0] ** 2))
backward_cd = float(np.mean(tree1.query(ref_points)[0] ** 2))
total_cd = forward_cd + backward_cd
assert total_cd <= cd_threshold, (
f"Chamfer Distance check failed: total_cd={total_cd:.6f}, "
f"threshold={cd_threshold:.6f} ({cd_threshold_ratio * 100:.2f}% of bbox diagonal {bbox_diagonal:.4f})"
)
# Registry of validators by name
VALIDATOR_REGISTRY = {
"default": PerformanceValidator,
"video": VideoPerformanceValidator,
"mesh": MeshValidator,
}
@@ -1095,7 +1188,103 @@ def get_generate_fn(
},
)
if modality == "video":
def generate_mesh(case_id, client) -> tuple[str, bytes]:
"""I2M: Image to Mesh generation using async /v1/meshes API."""
import requests as http_requests
if not sampling_params.image_path:
pytest.skip(f"{case_id}: no input image configured for mesh generation")
image_path = sampling_params.image_path
if isinstance(image_path, str) and is_image_url(image_path):
image_path = download_image_from_url(image_path)
elif isinstance(image_path, Path):
if not image_path.exists():
pytest.skip(f"{case_id}: image file missing: {image_path}")
else:
image_path = Path(str(image_path))
if not image_path.exists():
pytest.skip(f"{case_id}: image file missing: {image_path}")
base_url = str(client.base_url).rstrip("/")
if base_url.endswith("/v1"):
base_url = base_url[:-3]
create_url = f"{base_url}/v1/meshes"
with open(str(image_path), "rb") as img_file:
files = {"image": (Path(str(image_path)).name, img_file, "image/png")}
data = {
"prompt": "generate 3d mesh",
"model": model_path,
"seed": "0",
"guidance_scale": "5.0",
"num_inference_steps": "50",
}
logger.info(f"[Mesh Gen] Sending request to {create_url}")
try:
response = http_requests.post(
create_url, files=files, data=data, timeout=60
)
except Exception as e:
pytest.fail(f"{case_id}: mesh creation request failed: {e}")
if response.status_code != 200:
pytest.fail(f"{case_id}: mesh creation failed: {response.text}")
job = response.json()
mesh_id = job.get("id")
if not mesh_id:
pytest.fail(f"{case_id}: no mesh id in response: {job}")
poll_url = f"{base_url}/v1/meshes/{mesh_id}"
poll_interval = 5
max_wait = 1200
elapsed = 0
while elapsed < max_wait:
time.sleep(poll_interval)
elapsed += poll_interval
try:
poll_resp = http_requests.get(poll_url, timeout=30)
except Exception as e:
logger.warning(f"[Mesh Gen] Poll failed: {e}")
continue
if poll_resp.status_code != 200:
continue
status_data = poll_resp.json()
status = status_data.get("status", "")
if status == "completed":
content_url = f"{base_url}/v1/meshes/{mesh_id}/content"
try:
content_resp = http_requests.get(content_url, timeout=60)
except Exception as e:
pytest.fail(f"{case_id}: mesh download failed: {e}")
if content_resp.status_code != 200:
pytest.fail(f"{case_id}: mesh download failed: {content_resp.text}")
temp_path = Path(tempfile.gettempdir()) / f"mesh_test_{mesh_id}.glb"
temp_path.write_bytes(content_resp.content)
MESH_OUTPUT_PATHS[case_id] = str(temp_path)
logger.info(f"[Mesh Gen] Mesh downloaded to {temp_path}")
return (mesh_id, b"")
elif status == "failed":
error = status_data.get("error", {})
pytest.fail(f"{case_id}: mesh generation failed: {error}")
pytest.fail(f"{case_id}: mesh generation timed out after {max_wait}s")
if modality == "3d":
fn = generate_mesh
elif modality == "video":
if sampling_params.image_path and sampling_params.prompt:
if getattr(sampling_params, "direct_url_test", False):
fn = generate_text_url_image_to_video
python/sglang/multimodal_gen/test/server/testcase_configs.py
+20 -1
View File
@@ -190,6 +190,8 @@ class DiffusionServerArgs:
self.custom_validator = "image"
elif self.modality == "video":
self.custom_validator = "video"
elif self.modality == "3d":
self.custom_validator = "mesh"
@dataclass(frozen=True)
@@ -460,6 +462,11 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [
),
]
HUNYUAN3D_SHAPE_sampling_params = DiffusionSamplingParams(
prompt="",
image_path="https://raw.githubusercontent.com/sgl-project/sgl-test-files/main/diffusion-ci/consistency_gt/1-gpu/hunyuan3d_2_0/hunyuan3d.png",
)
ONE_GPU_CASES_B: list[DiffusionTestCase] = [
# === Text to Video (T2V) ===
DiffusionTestCase(
@@ -591,7 +598,19 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
),
]
# Skip turbowan because Triton requires 81920 shared memory, but AMD only has 65536.
# Skip hunyuan3d on AMD: marching_cubes surface extraction produces invalid SDF on ROCm.
if not current_platform.is_hip():
ONE_GPU_CASES_B.append(
DiffusionTestCase(
"hunyuan3d_shape_gen",
DiffusionServerArgs(
model_path="tencent/Hunyuan3D-2",
modality="3d",
),
HUNYUAN3D_SHAPE_sampling_params,
),
)
# Skip turbowan on AMD: Triton requires 81920 shared memory, but AMD only has 65536.
if not current_platform.is_hip():
ONE_GPU_CASES_B.append(
DiffusionTestCase(