Init attention backend for Intel XPU (#10656)
Co-authored-by: guangyey <guangye.yu@intel.com> Co-authored-by: DiweiSun <105627594+DiweiSun@users.noreply.github.com>
This commit is contained in:
co-authored by
guangyey
DiweiSun
parent
fb6cc7b000
commit
b113c72e7a
@@ -115,7 +115,7 @@ class RotaryEmbedding(CustomOp):
|
||||
if dtype == torch.float32 or (
|
||||
(not (_is_cuda or _is_npu) or self.head_size not in [64, 128, 256, 512])
|
||||
and not (_is_cpu and _is_cpu_amx_available)
|
||||
and not _is_xpu
|
||||
and not (_is_xpu)
|
||||
):
|
||||
from vllm._custom_ops import rotary_embedding
|
||||
|
||||
@@ -302,6 +302,7 @@ class RotaryEmbedding(CustomOp):
|
||||
offsets: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
# TODO: make a wrapper, and XPU will implement this kernel later.
|
||||
self.cos_sin_cache = self.cos_sin_cache.to(query.device)
|
||||
return self.forward_native(positions, query, key, offsets)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user