diff --git a/examples/python/CuTeDSL/ampere/call_with_tvm_ffi.py b/examples/python/CuTeDSL/ampere/call_with_tvm_ffi.py
deleted file mode 100644
index 39319a34..00000000
--- a/examples/python/CuTeDSL/ampere/call_with_tvm_ffi.py
+++ /dev/null
@@ -1,128 +0,0 @@
-# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: BSD-3-Clause
-
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-
-# 1. Redistributions of source code must retain the above copyright notice, this
-# list of conditions and the following disclaimer.
-
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-
-# 3. Neither the name of the copyright holder nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-import sys
-import os
-import torch
-import time
-
-import cutlass
-import cutlass.cute as cute
-from cutlass.cute.runtime import from_dlpack
-
-"""Demonstrates calling off-the-shelf kernels with TVM FFI without DLPack.
-
-This example shows how to compile CuTe JIT function with fake tensors then run it with TVM FFI.
-"""
-
-if __name__ == "__main__":
- # Add the current directory to sys.path
- current_dir = os.path.dirname(os.path.abspath(__file__))
- sys.path.insert(0, os.path.join(current_dir, ".."))
-from ampere.tensorop_gemm import TensorOpGemm
-
-
-def compile_op(use_tvm_ffi: bool = True):
- from cutlass.cute.runtime import make_fake_compact_tensor, make_fake_tensor
-
- a_shape = (cute.sym_int(), cute.sym_int(divisibility=16), cute.sym_int())
- b_shape = (cute.sym_int(), cute.sym_int(divisibility=16), cute.sym_int())
- c_shape = (cute.sym_int(), cute.sym_int(divisibility=16), cute.sym_int())
- a = make_fake_compact_tensor(
- cutlass.Float16, a_shape, stride_order=(1, 0, 2), assumed_align=16
- )
- b = make_fake_compact_tensor(
- cutlass.Float16, b_shape, stride_order=(1, 0, 2), assumed_align=16
- )
- c = make_fake_compact_tensor(
- cutlass.Float16, c_shape, stride_order=(1, 0, 2), assumed_align=16
- )
-
- tensor_op_gemm = TensorOpGemm(
- cutlass.Float16, cutlass.Float16, cutlass.Float32, (2, 2, 1)
- )
- compiled_fn = cute.compile(
- tensor_op_gemm, a, b, c, options="--enable-tvm-ffi" if use_tvm_ffi else ""
- )
- return compiled_fn
-
-
-def run_op(compiled_fn, mnkl, *, use_tvm_ffi: bool = True):
- print("\nRunning TensorOpGemm test with:")
- print(f"Tensor dimensions: {mnkl}")
- torch.manual_seed(1112)
- # (M,K,L)
- a_torch = torch.randn(
- mnkl[3], mnkl[0], mnkl[2], dtype=torch.float16, device="cuda"
- ).permute(1, 2, 0)
- # (N,K,L)
- b_torch = torch.randn(
- mnkl[3], mnkl[1], mnkl[2], dtype=torch.float16, device="cuda"
- ).permute(1, 2, 0)
- # (N,M,L)
- c_torch = torch.randn(
- mnkl[3], mnkl[0], mnkl[1], dtype=torch.float16, device="cuda"
- ).permute(1, 2, 0)
-
- print("Input tensor shapes:")
- print(f"a: {a_torch.shape}, dtype: {a_torch.dtype}")
- print(f"b: {b_torch.shape}, dtype: {b_torch.dtype}")
- print(f"c: {c_torch.shape}, dtype: {c_torch.dtype}\n")
- if not use_tvm_ffi:
- a = from_dlpack(a_torch).mark_layout_dynamic(leading_dim=1)
- b = from_dlpack(b_torch).mark_layout_dynamic(leading_dim=1)
- c = from_dlpack(c_torch).mark_layout_dynamic(leading_dim=1)
- else:
- a = a_torch
- b = b_torch
- c = c_torch
- # pass in torch tensor as input
- compiled_fn(a, b, c)
- torch.cuda.synchronize()
-
- # measure the launch overhead of tvm ffi function
- repeat = 100
- start_time = time.time()
- for i in range(repeat):
- compiled_fn(a, b, c)
- end_time = time.time()
- print(
- f"Launch overhead of tvm ffi function: {(end_time - start_time) / repeat} seconds"
- )
-
- ref = torch.einsum("mkl,nkl->mnl", a_torch, b_torch)
- torch.testing.assert_close(c_torch, ref, atol=1e-05, rtol=1e-05)
- print("\n[DSL INFO] Results verified successfully!")
- print(f"First few elements of result: \n{c_torch[:3, :3, :3]}")
-
-
-if __name__ == "__main__":
- compiled_fn = compile_op(use_tvm_ffi=False)
- run_op(compiled_fn, [512, 512, 256, 1], use_tvm_ffi=False)
- compiled_fn = compile_op(use_tvm_ffi=True)
- run_op(compiled_fn, [512, 512, 256, 1], use_tvm_ffi=True)
diff --git a/examples/python/CuTeDSL/blackwell/dense_gemm_persistent.py b/examples/python/CuTeDSL/blackwell/dense_gemm_persistent.py
index d5b48f45..3fdd1618 100644
--- a/examples/python/CuTeDSL/blackwell/dense_gemm_persistent.py
+++ b/examples/python/CuTeDSL/blackwell/dense_gemm_persistent.py
@@ -34,6 +34,7 @@ import cuda.bindings.driver as cuda
import cutlass
import cutlass.cute as cute
import cutlass.cute.testing as testing
+from cutlass.cute.runtime import from_dlpack
import cutlass.utils as utils
import cutlass.pipeline as pipeline
from cutlass.pipeline import pipeline_init_arrive, pipeline_init_wait
@@ -1704,55 +1705,6 @@ def bmm(
gemm_op(a, b, c, max_active_clusters, stream, epilogue_op)
-def compile_bmm(
- gemm_op: PersistentDenseGemmKernel,
- a_dtype: Type[cutlass.Numeric],
- b_dtype: Type[cutlass.Numeric],
- c_dtype: Type[cutlass.Numeric],
- a_major: str,
- b_major: str,
- c_major: str,
- max_active_clusters: cutlass.Constexpr,
- stream: cuda.CUstream,
- epilogue_op: cutlass.Constexpr = lambda x: x,
- options: str = "",
-):
- from cutlass.cute.runtime import make_fake_compact_tensor
-
- a_shape = (cute.sym_int(), cute.sym_int(divisibility=16), cute.sym_int())
- b_shape = (cute.sym_int(), cute.sym_int(divisibility=16), cute.sym_int())
- c_shape = (cute.sym_int(), cute.sym_int(divisibility=16), cute.sym_int())
-
- if a_major == "k":
- a_order = (2, 1, 0) # k is leading dimension
- elif a_major == "m":
- a_order = (2, 0, 1) # m is leading dimension
-
- if b_major == "n":
- b_order = (2, 1, 0) # n is leading dimension
- elif b_major == "k":
- b_order = (2, 0, 1) # k is leading dimension
-
- if c_major == "n":
- c_order = (2, 1, 0) # n is leading dimension
- elif c_major == "m":
- c_order = (2, 0, 1) # m is leading dimension
-
- a = make_fake_compact_tensor(
- a_dtype, a_shape, stride_order=a_order, assumed_align=16
- )
- b = make_fake_compact_tensor(
- b_dtype, b_shape, stride_order=b_order, assumed_align=16
- )
- c = make_fake_compact_tensor(
- c_dtype, c_shape, stride_order=c_order, assumed_align=16
- )
-
- return cute.compile(
- bmm, gemm_op, a, b, c, max_active_clusters, stream, epilogue_op, options=options
- )
-
-
def prepare_tensors(
mnkl: Tuple[int, int, int, int],
ab_dtype: Type[cutlass.Numeric],
@@ -1811,7 +1763,6 @@ def run(
iterations: int = 1,
skip_ref_check: bool = False,
use_cold_l2: bool = False,
- use_tvm_ffi: bool = False,
benchmark: bool = False,
**kwargs,
):
@@ -1853,8 +1804,6 @@ def run(
:type skip_ref_check: bool, optional
:param use_cold_l2: Whether to use circular buffer strategy to ensure cold L2 cache, defaults to False.
:type use_cold_l2: bool, optional
- :param use_tvm_ffi: Whether to use TVM FFI for the kernel, defaults to False.
- :type use_tvm_ffi: bool, optional
:param benchmark: Whether to only benchmark the kernel, defaults to False.
:type benchmark: bool, optional
:raises RuntimeError: If CUDA GPU is not available.
@@ -1874,16 +1823,15 @@ def run(
print(f"Iterations: {iterations}")
print(f"Skip reference checking: {skip_ref_check}")
print(f"Use cold L2: {'True' if use_cold_l2 else 'False'}")
- print(f"Use TVM FFI: {'True' if use_tvm_ffi else 'False'}")
import torch
from cutlass.torch import dtype as torch_dtype
# Build GEMM object
- gemm = PersistentDenseGemmKernel(
+ gemm_op = PersistentDenseGemmKernel(
acc_dtype, use_2cta_instrs, mma_tiler_mn, cluster_shape_mn, use_tma_store
)
- can_implement = gemm.can_implement(
+ can_implement = gemm_op.can_implement(
mnkl, ab_dtype, c_dtype, a_major, b_major, c_major
)
if not can_implement:
@@ -1906,29 +1854,32 @@ def run(
cluster_shape_mn[0] * cluster_shape_mn[1]
)
- options = []
- if use_tvm_ffi:
- options.append("--enable-tvm-ffi")
-
- compiled_fn = compile_bmm(
- gemm,
- ab_dtype,
- ab_dtype,
- c_dtype,
- a_major,
- b_major,
- c_major,
- max_active_clusters,
- current_stream,
- options=",".join(options),
- )
-
# Run and verify BMM with torch
a, b, c = prepare_tensors(mnkl, ab_dtype, c_dtype, a_major, b_major, c_major)
+ # Leading dim is 2
+ leading_dim_a = 2 if a_major == "k" else 1
+ leading_dim_b = 1 if b_major == "k" else 2
+ leading_dim_c = 2 if c_major == "n" else 1
+
+ a_ = from_dlpack(a).mark_layout_dynamic(leading_dim=leading_dim_a)
+ b_ = from_dlpack(b).mark_layout_dynamic(leading_dim=leading_dim_b)
+ c_ = from_dlpack(c).mark_layout_dynamic(leading_dim=leading_dim_c)
+
+ compiled_fn = cute.compile(
+ bmm,
+ gemm_op,
+ a_,
+ b_,
+ c_,
+ max_active_clusters,
+ current_stream,
+ epilogue_op=lambda x: x,
+ )
+
if not skip_ref_check:
# Use small random number for deterministic result for reference check
- compiled_fn(a, b, c, torch_stream)
+ compiled_fn(a_, b_, c_, current_stream)
# Manually quantize to be comparable
ref = (
@@ -1954,7 +1905,10 @@ def run(
c_major,
init_random=not init_normal,
)
- return testing.JitArguments(a, b, c, torch_stream)
+ a_ = from_dlpack(a).mark_layout_dynamic(leading_dim=leading_dim_a)
+ b_ = from_dlpack(b).mark_layout_dynamic(leading_dim=leading_dim_b)
+ c_ = from_dlpack(c).mark_layout_dynamic(leading_dim=leading_dim_c)
+ return testing.JitArguments(a_, b_, c_, current_stream)
workspace_count = 1
if use_cold_l2:
@@ -2043,12 +1997,6 @@ def prepare_parser():
default=False,
help="Use circular buffer tensor sets to ensure L2 cold cache",
)
- parser.add_argument(
- "--use_tvm_ffi",
- action="store_true",
- default=False,
- help="Enable TVM FFI for the kernel, defaults to False using CuTe DSL's native runtime",
- )
return parser
@@ -2090,7 +2038,6 @@ if __name__ == "__main__":
args.iterations,
args.skip_ref_check,
args.use_cold_l2,
- args.use_tvm_ffi,
args.benchmark,
)
print("PASS")
diff --git a/examples/python/CuTeDSL/cute/tvm_ffi/ampere_gemm_with_fake_tensor.py b/examples/python/CuTeDSL/cute/tvm_ffi/ampere_gemm_with_fake_tensor.py
new file mode 100644
index 00000000..68e8f81a
--- /dev/null
+++ b/examples/python/CuTeDSL/cute/tvm_ffi/ampere_gemm_with_fake_tensor.py
@@ -0,0 +1,170 @@
+# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: BSD-3-Clause
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+
+# 3. Neither the name of the copyright holder nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import sys
+import os
+import torch
+
+import cutlass
+import cutlass.cute as cute
+from cutlass.cute.runtime import from_dlpack
+
+"""Demonstrates calling off-the-shelf kernels with TVM FFI without DLPack.
+
+This example shows how to compile CuTe JIT function with fake tensors then run it with TVM FFI.
+"""
+
+if __name__ == "__main__":
+ # Add the current directory to sys.path
+ current_dir = os.path.dirname(os.path.abspath(__file__))
+ sys.path.insert(0, os.path.join(current_dir, "..", ".."))
+from ampere.tensorop_gemm import TensorOpGemm
+
+
+@cute.jit
+def bmm(
+ a: cute.Tensor, # (l, m, k)
+ b: cute.Tensor, # (l, k, n)
+ c: cute.Tensor, # (l, m, n)
+):
+ gemm_op = TensorOpGemm(cutlass.Float16, cutlass.Float16, cutlass.Float32, (2, 2, 1))
+
+ # Permute to follow convention of CuTe
+
+ # (l, m, k) -> (m, k, l)
+ a = cute.make_tensor(a.iterator, cute.select(a.layout, mode=[1, 2, 0]))
+ # (l, k, n) -> (n, k, l)
+ b = cute.make_tensor(b.iterator, cute.select(b.layout, mode=[2, 1, 0]))
+ # (l, m, n) -> (m, n, l)
+ c = cute.make_tensor(c.iterator, cute.select(c.layout, mode=[1, 2, 0]))
+
+ gemm_op(a, b, c)
+
+
+def compile_bmm_dynamic_layout():
+ from cutlass.cute.runtime import make_fake_compact_tensor
+
+ m = cute.sym_int()
+ n = cute.sym_int(divisibility=16)
+ k = cute.sym_int(divisibility=16)
+ l = cute.sym_int()
+
+ # Contiguous on K
+ fake_a = make_fake_compact_tensor(
+ cutlass.Float16, (l, m, k), stride_order=(2, 1, 0), assumed_align=16
+ )
+ # Contiguous on N
+ fake_b = make_fake_compact_tensor(
+ cutlass.Float16, (l, k, n), stride_order=(2, 1, 0), assumed_align=16
+ )
+ # Contiguous on N
+ fake_c = make_fake_compact_tensor(
+ cutlass.Float16, (l, m, n), stride_order=(2, 1, 0), assumed_align=16
+ )
+
+ compiled_fn = cute.compile(bmm, fake_a, fake_b, fake_c, options="--enable-tvm-ffi")
+ return compiled_fn
+
+
+def compile_bmm_static_layout(m, n, k, l):
+ from cutlass.cute.runtime import make_fake_compact_tensor
+
+ fake_a = make_fake_compact_tensor(
+ cutlass.Float16, (l, m, k), stride_order=(2, 1, 0), assumed_align=16
+ )
+ fake_b = make_fake_compact_tensor(
+ cutlass.Float16, (l, k, n), stride_order=(2, 1, 0), assumed_align=16
+ )
+ fake_c = make_fake_compact_tensor(
+ cutlass.Float16, (l, m, n), stride_order=(2, 1, 0), assumed_align=16
+ )
+
+ compiled_fn = cute.compile(bmm, fake_a, fake_b, fake_c, options="--enable-tvm-ffi")
+ return compiled_fn
+
+
+def run_bmm_and_verify(compiled_fn, m, n, k, l):
+ torch.manual_seed(1112)
+
+ a = torch.randn(l, m, k, dtype=torch.float16, device="cuda")
+ b = torch.randn(l, k, n, dtype=torch.float16, device="cuda")
+ c = torch.randn(l, m, n, dtype=torch.float16, device="cuda")
+
+ print("[Runtime INFO] Input tensor shapes:")
+ print(f"a: {a.shape=}, {a.stride()=}, {a.dtype=}")
+ print(f"b: {b.shape=}, {b.stride()=}, {b.dtype=}")
+ print(f"c: {c.shape=}, {c.stride()=}, {c.dtype=}\n")
+
+ # pass in torch tensor as input
+ compiled_fn(a, b, c)
+ torch.cuda.synchronize()
+
+ ref = torch.bmm(a, b)
+ torch.testing.assert_close(c, ref, atol=1e-05, rtol=1e-05)
+ print("[Runtime INFO] Verification successful!")
+ print(f" First few elements of result: \n{c[:3, :3, :3]}")
+
+
+if __name__ == "__main__":
+ m, n, k, l = (512, 512, 256, 2)
+
+ compiled_fn_dynamic = compile_bmm_dynamic_layout()
+ run_bmm_and_verify(compiled_fn_dynamic, m, n, k, l)
+
+ compiled_fn_static = compile_bmm_static_layout(m, n, k, l)
+ run_bmm_and_verify(compiled_fn_static, m, n, k, l)
+
+ # Error Check:
+ # 1. mis-matched tensor dim raise error
+ a = torch.randn(l, m, k, dtype=torch.float16, device="cuda")
+ b = torch.randn(l, 2 * k, n, dtype=torch.float16, device="cuda")
+ c = torch.randn(l, m, n, dtype=torch.float16, device="cuda")
+ try:
+ compiled_fn_dynamic(a, b, c)
+ except Exception as e:
+ print(f"\n[Runtime Error]: {e}")
+
+ # 2. mis-matched divisibility
+ a = torch.randn(l, m, k + 1, dtype=torch.float16, device="cuda")
+ b = torch.randn(l, k + 1, n, dtype=torch.float16, device="cuda")
+ c = torch.randn(l, m, n, dtype=torch.float16, device="cuda")
+
+ try:
+ compiled_fn_dynamic(a, b, c)
+ except Exception as e:
+ print(f"\n[Runtime Error]: {e}")
+
+ # 3. mis-matched static shape constraint
+ a = torch.randn(l * 2, m, k, dtype=torch.float16, device="cuda")
+ b = torch.randn(l * 2, k, n, dtype=torch.float16, device="cuda")
+ c = torch.randn(l * 2, m, n, dtype=torch.float16, device="cuda")
+
+ try:
+ compiled_fn_static(a, b, c)
+ except Exception as e:
+ print(f"\n[Runtime Error]: {e}")
diff --git a/examples/python/CuTeDSL/cute/fake_tensor.py b/examples/python/CuTeDSL/cute/tvm_ffi/compile_with_fake_tensor.py
similarity index 78%
rename from examples/python/CuTeDSL/cute/fake_tensor.py
rename to examples/python/CuTeDSL/cute/tvm_ffi/compile_with_fake_tensor.py
index fa53c506..1b001ce3 100644
--- a/examples/python/CuTeDSL/cute/fake_tensor.py
+++ b/examples/python/CuTeDSL/cute/tvm_ffi/compile_with_fake_tensor.py
@@ -22,26 +22,26 @@ def run():
shape = (3, 4)
a = make_fake_compact_tensor(cutlass.Float16, (3, 4), stride_order=(1, 0))
- cute.compile(print_tensor_type, a)
+ cute.compile(print_tensor_type, a, options="--enable-tvm-ffi")
# 32-bit symbolic integer with divisibility 8
shape = (3, cute.sym_int32(divisibility=8))
a = make_fake_compact_tensor(cutlass.Float16, shape, stride_order=(1, 0))
- cute.compile(print_tensor_type, a)
+ cute.compile(print_tensor_type, a, options="--enable-tvm-ffi")
# with static stride
a = make_fake_tensor(cutlass.Float16, shape, stride=(4, 1))
- cute.compile(print_tensor_type, a)
+ cute.compile(print_tensor_type, a, options="--enable-tvm-ffi")
# with dynamic stride using 32bit integer
stride = (cute.sym_int32(divisibility=8), 1)
a = make_fake_tensor(cutlass.Float16, shape, stride=stride)
- cute.compile(print_tensor_type, a)
+ cute.compile(print_tensor_type, a, options="--enable-tvm-ffi")
# with dynamic stride using 64bit integer
stride = (cute.sym_int64(divisibility=8), 1)
a = make_fake_tensor(cutlass.Float16, shape, stride=stride)
- cute.compile(print_tensor_type, a)
+ cute.compile(print_tensor_type, a, options="--enable-tvm-ffi")
if __name__ == "__main__":
diff --git a/examples/python/CuTeDSL/cute/tvm_ffi/requirements.txt b/examples/python/CuTeDSL/cute/tvm_ffi/requirements.txt
new file mode 100644
index 00000000..8a3f9dcd
--- /dev/null
+++ b/examples/python/CuTeDSL/cute/tvm_ffi/requirements.txt
@@ -0,0 +1,2 @@
+apache-tvm-ffi
+torch-c-dlpack-ext
\ No newline at end of file
diff --git a/media/docs/pythonDSL/cute_dsl_general/compile_with_tvm_ffi.rst b/media/docs/pythonDSL/cute_dsl_general/compile_with_tvm_ffi.rst
index 42d0fdea..b3d49af7 100644
--- a/media/docs/pythonDSL/cute_dsl_general/compile_with_tvm_ffi.rst
+++ b/media/docs/pythonDSL/cute_dsl_general/compile_with_tvm_ffi.rst
@@ -4,7 +4,8 @@
Compile with TVM FFI
====================
-Apache TVM FFI is an open ABI and FFI for machine learning systems. More information can be found in the `official documentation `_.
+Apache TVM FFI is an open ABI and FFI for machine learning systems. More information can be found in
+the `official documentation `_.
To install TVM FFI, you can run the following command:
@@ -14,7 +15,9 @@ To install TVM FFI, you can run the following command:
# optional package for improved torch tensor calling performance
pip install torch-c-dlpack-ext
-In |DSL|, TVM FFI can be enabled as an option for JIT-compiled functions. Using TVM FFI can lead to faster JIT function invocation and provides better interoperability with machine learning frameworks (e.g., directly take ``torch.Tensor`` as arguments).
+In |DSL|, TVM FFI can be enabled as an option for JIT-compiled functions. Using TVM FFI can lead to faster
+JIT function invocation and provides better interoperability with machine learning frameworks
+(e.g., directly take ``torch.Tensor`` as arguments).
Enable Apache TVM FFI in |DSL|
@@ -40,7 +43,8 @@ There are two ways to enable TVM FFI in |DSL|:
Note that the object returned by ``cute.compile`` is a Python function specific to TVM FFI.
-2. Alternatively, you can enable TVM FFI globally by setting the environment variable ``CUTE_DSL_ENABLE_TVM_FFI=1``. Please note that this setting will apply to all JIT compilations within the environment.
+2. Alternatively, you can enable TVM FFI globally by setting the environment variable ``CUTE_DSL_ENABLE_TVM_FFI=1``.
+Please note that this setting will apply to all JIT compilations within the environment.
Minimizing Host Overhead
@@ -114,6 +118,37 @@ The fake tensor is a placeholder that mimics the interface of a real tensor but
It is used in compilation or testing scenarios where only shape/type/layout information is needed.
All attempts to access or mutate data will raise errors.
+
+Interoperability with `from_dlpack`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The Fake Tensor flow supports more flexible constraints on Tensor arguments than the `from_dlpack` flow.
+When fake tensor is used, it's recommended to use TVM FFI backend as it supports more flexible constraints on
+Tensor arguments than the `from_dlpack` flow.
+
+For instance, fake tensor can specify per-mode static shape or constraints on shape and strides which is not supported by
+`from_dlpack`. It's expected that JIT function compiled with fake tensor may have different ABI with tensor converted
+with `from_dlpack`.
+
+.. code-block:: python
+
+ import cutlass.cute as cute
+ import torch
+
+ n = cute.sym_int()
+ # Dynamic Shape
+ fake_a = cute.runtime.make_fake_compact_tensor(cute.Float32, (n,))
+
+ # Compile without tvm-ffi
+ compiled_fn = cute.compile(foo, fake_a)
+
+ # Wrong, in compatible ABI
+ compiled_fn(from_dlpack(a))
+
+
+In order to avoid mismatched ABI, it's recommended to use TVM FFI when fake tensor is used for compilation.
+
+
Note on Stride Order
~~~~~~~~~~~~~~~~~~~~
@@ -129,7 +164,8 @@ stride via the ``stride`` argument in the ``make_fake_tensor`` API.
``cute.Tensor`` adapter for TVM FFI
-----------------------------------
-To adapt the ``cute.Tensor`` to the TVM FFI function, you can use the ``cute.runtime.from_dlpack`` function with the ``enable_tvm_ffi=True`` option or the environment variable ``CUTE_DSL_ENABLE_TVM_FFI=1``. For example:
+To adapt the ``cute.Tensor`` to the TVM FFI function, you can use the ``cute.runtime.from_dlpack`` function with the
+``enable_tvm_ffi=True`` option or the environment variable ``CUTE_DSL_ENABLE_TVM_FFI=1``. For example:
.. code-block:: python
@@ -288,6 +324,7 @@ composed of the types that are supported by TVM FFI. The example below shows how
example_add_one_with_tuple()
+
Supported types
---------------
@@ -316,6 +353,7 @@ The TVM FFI function supports the following |DSL|-specific types as arguments:
* - Tuple of types (e.g. ``Tuple[cute.Tensor, cute.Tensor, cutlass.Int32]``)
- Python tuple of corresponding call-time types.
+
Error handling
--------------
diff --git a/python/CuTeDSL/cutlass/cute/runtime.py b/python/CuTeDSL/cutlass/cute/runtime.py
index 19815b5c..07fe03d0 100644
--- a/python/CuTeDSL/cutlass/cute/runtime.py
+++ b/python/CuTeDSL/cutlass/cute/runtime.py
@@ -617,7 +617,8 @@ def make_fake_compact_tensor(
:param shape: Shape of the tensor.
:type shape: tuple[int, ...]
:param stride_order: Order in which strides (memory layout) are assigned to the tensor dimensions.
- If None, the default layout is col-major. Otherwise, it should be a permutation of the dimension indices.
+ If None, the default layout is left-to-right order (known as column-major order for flatten layout).
+ Otherwise, it should be a permutation order of the dimension indices.
:type stride_order: tuple[int, ...], optional
:param memspace: Memory space where the fake tensor resides. Optional.
:type memspace: str, optional