diff --git a/README.md b/README.md index 911d859b7..99c24348b 100755 --- a/README.md +++ b/README.md @@ -790,6 +790,28 @@ We added ring attention support for Wan models. Below are the stats for one `720 (* There are some known stability issues for ring attention on 16 TPUs, please use `tokamax_flash` attention instead.) +### FlyWheel Attention +`attention=flywheel` runs Wan self-attention on the [FlyWheel](https://github.com/heavyball-research/flywheel-tpu) kernel instead of Splash, with the same sharding as `attention=flash`. `attention=ulysses_flywheel` runs the same kernel with the head sharding of `attention=ulysses` (requires `ici_context_parallelism` > 1 and heads divisible by it). Cross-attention, RoPE and QK-norm are unchanged and stay on Splash/XLA. + +FlyWheel supports TPU v6e and tpu7x, and it is inference-only: the kernel has no backward pass, so do not use it for training. Inputs it does not support (attention masks, causal attention, GQA, non-bfloat16 q/k/v) raise an error instead of falling back to another kernel. + +```bash +pip install flywheel-tpu # or: pip install -e ".[flywheel]" +python src/maxdiffusion/generate_wan.py \ + src/maxdiffusion/configs/base_wan_27b.yml \ + attention="flywheel" \ + ... +``` + +Wan 2.1 T2V 1.3B, `480x832`, 81 frames, 30 inference steps, CFG, seed 0, one v6e chip. Splash uses the faster of two v6e tilings we tried (`block_q`/`block_kv`/`block_kv_compute` 2048/4096/1024; 3024/2048/1024 took 65.1 s). Times are the average of two runs, which differed by less than 0.2%. + +| Accelerator | Model | Attention Type | Denoise | e2e Generation Time | +| -- | -- | -- | -- | -- | +| v6e-1 | WAN 2.1 1.3B | Splash (`flash`) | 58.8 s | 63.3 s | +| v6e-1 | WAN 2.1 1.3B | FlyWheel (`flywheel`) | **49.5 s** | **54.0 s (1.17x)** | + +The FlyWheel video differs from the Splash video by 25.0 dB PSNR, about the same as changing only the Splash tiling (25.1 dB between the 512 and 2048/4096/1024 tilings). + ### Automatic Tile-Size Search The optimal attention tile sizes (`block_q` / `block_kv`) depend on the sequence length, VMEM, sharding, and accelerator, and the feasibility edge is a VMEM OOM with no clean closed form — so we tune them empirically. Passing `enable_tile_search=true` to `generate_wan.py` runs a fast one-DiT-block grid search before inference and injects the winning block sizes into `flash_block_sizes` (`tile_search_mode=smart` by default; the search is opt-in and off by default). The core is model-agnostic (`utils/tile_size_grid_search.py`) with a per-model plug (`utils/wan_block_benchmark.py`), which also runs standalone via `python -m maxdiffusion.utils.wan_block_benchmark ... --smart-search`. diff --git a/end_to_end/tpu/run_wan_stack_tests.sh b/end_to_end/tpu/run_wan_stack_tests.sh index eb4400dc6..ed979f803 100755 --- a/end_to_end/tpu/run_wan_stack_tests.sh +++ b/end_to_end/tpu/run_wan_stack_tests.sh @@ -45,5 +45,7 @@ TESTS=( # Fused RMSNorm+RoPE producer, Wan switches in config (feat/wan-custom-kernels) "$T/fused_rmsnorm_rope_pallas_test.py" "$T/wan_runtime_options_test.py" + # FlyWheel self-attention backend (needs flywheel-tpu on v6e / tpu7x for the numerical tests) + "$T/flywheel_attention_test.py" ) PYTHONPATH="src${PYTHONPATH:+:$PYTHONPATH}" exec "${PYTHON:-python3}" -m pytest -q -rs "${TESTS[@]}" "$@" diff --git a/pyproject.toml b/pyproject.toml index 25d1cc31e..929ff797e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,8 @@ classifiers = [ quality = ["urllib3", "black", "isort", "ruff", "hf-doc-builder"] docs = ["hf-doc-builder"] training = ["accelerate", "datasets", "protobuf", "tensorboard", "Jinja2"] +# attention=flywheel / ulysses_flywheel (Wan inference on TPU v6e and tpu7x). +flywheel = ["flywheel-tpu>=0.4.1"] test = [ "compel", "datasets", diff --git a/src/maxdiffusion/configs/base_wan_14b.yml b/src/maxdiffusion/configs/base_wan_14b.yml index 5404f1a22..ebede599f 100644 --- a/src/maxdiffusion/configs/base_wan_14b.yml +++ b/src/maxdiffusion/configs/base_wan_14b.yml @@ -101,7 +101,7 @@ svg_low_noise_density: -1.0 # compute blocks and the sparse kernel for small ones. Must stay a dict so the # command line can override it with JSON. svg_flash_block_sizes: {} -attention: 'flash' # Supported attention: dot_product, flash, tokamax_flash, cudnn_flash_te, ring, tokamax_ring, tokamax_ring_custom, ulysses, ulysses_custom, ulysses_custom_fixed_m, ulysses_custom_fixed_m_per_q_block, ulysses_ring, ulysses_ring_custom, ulysses_ring_custom_fixed_m, ulysses_ring_custom_fixed_m_per_q_block, ulysses_ring_custom_bidir +attention: 'flash' # Supported attention: dot_product, flash, flywheel, ulysses_flywheel, tokamax_flash, cudnn_flash_te, ring, tokamax_ring, tokamax_ring_custom, ulysses, ulysses_custom, ulysses_custom_fixed_m, ulysses_custom_fixed_m_per_q_block, ulysses_ring, ulysses_ring_custom, ulysses_ring_custom_fixed_m, ulysses_ring_custom_fixed_m_per_q_block, ulysses_ring_custom_bidir use_base2_exp: True use_experimental_scheduler: True diff --git a/src/maxdiffusion/configs/base_wan_1_3b.yml b/src/maxdiffusion/configs/base_wan_1_3b.yml index cfb8c9a4d..22f66d2bb 100644 --- a/src/maxdiffusion/configs/base_wan_1_3b.yml +++ b/src/maxdiffusion/configs/base_wan_1_3b.yml @@ -98,7 +98,7 @@ svg_low_noise_density: -1.0 # compute blocks and the sparse kernel for small ones. Must stay a dict so the # command line can override it with JSON. svg_flash_block_sizes: {} -attention: 'flash' # Supported attention: dot_product, flash, tokamax_flash, cudnn_flash_te, ring, tokamax_ring, ulysses, ulysses_custom, ulysses_ring +attention: 'flash' # Supported attention: dot_product, flash, flywheel, ulysses_flywheel, tokamax_flash, cudnn_flash_te, ring, tokamax_ring, ulysses, ulysses_custom, ulysses_ring use_base2_exp: True use_experimental_scheduler: True diff --git a/src/maxdiffusion/configs/base_wan_27b.yml b/src/maxdiffusion/configs/base_wan_27b.yml index a6872a911..a193dc3ca 100644 --- a/src/maxdiffusion/configs/base_wan_27b.yml +++ b/src/maxdiffusion/configs/base_wan_27b.yml @@ -103,7 +103,7 @@ svg_low_noise_density: -1.0 # {"block_q":3328,"block_kv":2816,"block_kv_compute":256, # "block_kv_compute_in":256,"heads_per_tile":1,"vmem_limit_bytes":67108864}. svg_flash_block_sizes: {} -attention: 'flash' # Supported attention: dot_product, flash, tokamax_flash, cudnn_flash_te, ring, tokamax_ring, tokamax_ring_custom, ulysses, ulysses_custom, ulysses_custom_fixed_m, ulysses_custom_fixed_m_per_q_block, ulysses_ring, ulysses_ring_custom, ulysses_ring_custom_fixed_m, ulysses_ring_custom_fixed_m_per_q_block, ulysses_ring_custom_bidir +attention: 'flash' # Supported attention: dot_product, flash, flywheel, ulysses_flywheel, tokamax_flash, cudnn_flash_te, ring, tokamax_ring, tokamax_ring_custom, ulysses, ulysses_custom, ulysses_custom_fixed_m, ulysses_custom_fixed_m_per_q_block, ulysses_ring, ulysses_ring_custom, ulysses_ring_custom_fixed_m, ulysses_ring_custom_fixed_m_per_q_block, ulysses_ring_custom_bidir # diff --git a/src/maxdiffusion/configs/base_wan_animate.yml b/src/maxdiffusion/configs/base_wan_animate.yml index b53904424..d7d1f6f01 100644 --- a/src/maxdiffusion/configs/base_wan_animate.yml +++ b/src/maxdiffusion/configs/base_wan_animate.yml @@ -81,7 +81,7 @@ jit_initializers: True # Set true to load weights from pytorch from_pt: True split_head_dim: False -attention: 'flash' # Supported attention: dot_product, flash, tokamax_flash, cudnn_flash_te, ring, tokamax_ring, ulysses, ulysses_custom, ulysses_custom_fixed_m, ulysses_custom_fixed_m_per_q_block, ulysses_ring, ulysses_ring_custom, ulysses_ring_custom_fixed_m, ulysses_ring_custom_fixed_m_per_q_block +attention: 'flash' # Supported attention: dot_product, flash, flywheel, ulysses_flywheel, tokamax_flash, cudnn_flash_te, ring, tokamax_ring, ulysses, ulysses_custom, ulysses_custom_fixed_m, ulysses_custom_fixed_m_per_q_block, ulysses_ring, ulysses_ring_custom, ulysses_ring_custom_fixed_m, ulysses_ring_custom_fixed_m_per_q_block use_base2_exp: True use_experimental_scheduler: True # use_k_centering: "auto" turns K-centering on for the non-ring diff --git a/src/maxdiffusion/configs/base_wan_i2v_14b.yml b/src/maxdiffusion/configs/base_wan_i2v_14b.yml index 0c403f7e5..acb09cd02 100644 --- a/src/maxdiffusion/configs/base_wan_i2v_14b.yml +++ b/src/maxdiffusion/configs/base_wan_i2v_14b.yml @@ -83,7 +83,7 @@ jit_initializers: True # Set true to load weights from pytorch from_pt: True split_head_dim: False -attention: 'flash' # Supported attention: dot_product, flash, tokamax_flash, cudnn_flash_te, ring, tokamax_ring, ulysses, ulysses_custom, ulysses_custom_fixed_m, ulysses_custom_fixed_m_per_q_block, ulysses_ring, ulysses_ring_custom, ulysses_ring_custom_fixed_m, ulysses_ring_custom_fixed_m_per_q_block +attention: 'flash' # Supported attention: dot_product, flash, flywheel, ulysses_flywheel, tokamax_flash, cudnn_flash_te, ring, tokamax_ring, ulysses, ulysses_custom, ulysses_custom_fixed_m, ulysses_custom_fixed_m_per_q_block, ulysses_ring, ulysses_ring_custom, ulysses_ring_custom_fixed_m, ulysses_ring_custom_fixed_m_per_q_block use_base2_exp: True use_experimental_scheduler: True # use_k_centering: "auto" turns K-centering on for the non-ring diff --git a/src/maxdiffusion/configs/base_wan_i2v_27b.yml b/src/maxdiffusion/configs/base_wan_i2v_27b.yml index 0812feefe..8f53ec13f 100644 --- a/src/maxdiffusion/configs/base_wan_i2v_27b.yml +++ b/src/maxdiffusion/configs/base_wan_i2v_27b.yml @@ -83,7 +83,7 @@ jit_initializers: True # Set true to load weights from pytorch from_pt: True split_head_dim: False -attention: 'flash' # Supported attention: dot_product, flash, tokamax_flash, cudnn_flash_te, ring, tokamax_ring, ulysses, ulysses_custom, ulysses_custom_fixed_m, ulysses_custom_fixed_m_per_q_block, ulysses_ring, ulysses_ring_custom, ulysses_ring_custom_fixed_m, ulysses_ring_custom_fixed_m_per_q_block +attention: 'flash' # Supported attention: dot_product, flash, flywheel, ulysses_flywheel, tokamax_flash, cudnn_flash_te, ring, tokamax_ring, ulysses, ulysses_custom, ulysses_custom_fixed_m, ulysses_custom_fixed_m_per_q_block, ulysses_ring, ulysses_ring_custom, ulysses_ring_custom_fixed_m, ulysses_ring_custom_fixed_m_per_q_block use_base2_exp: True use_experimental_scheduler: True # use_k_centering: "auto" turns K-centering on for the non-ring diff --git a/src/maxdiffusion/models/attention_flax.py b/src/maxdiffusion/models/attention_flax.py index d1d9937e3..d0fa35882 100644 --- a/src/maxdiffusion/models/attention_flax.py +++ b/src/maxdiffusion/models/attention_flax.py @@ -1051,6 +1051,118 @@ def ring_scan_body(carry, _): return x +# --------------------------------------------------------------------------- +# FlyWheel attention (https://github.com/heavyball-research/flywheel-tpu) +# --------------------------------------------------------------------------- + +FLYWHEEL_KERNELS = ("flywheel", "ulysses_flywheel") + + +def _flywheel_attention( + query: jax.Array, + key: jax.Array, + value: jax.Array, + heads: int, + mesh: Mesh, + axis_names_q: AxisNames, + axis_names_kv: AxisNames, + scale: float, + use_ulysses: bool = False, + attention_mask: jax.Array = None, + is_causal: bool = False, + kv_heads: Optional[int] = None, + ulysses_shards: int = -1, + ulysses_attention_chunks: int = 1, +) -> jax.Array: + """FlyWheel `flash_attn_varlen_func` self-attention. + + `use_ulysses=False` keeps the `flash` sharding (sequence-sharded q, full K/V); + `use_ulysses=True` keeps the `ulysses` sharding (all-to-all around the kernel). + Each device makes one varlen call with cu_seqlens = [0, length], so K/V padding + is never attended. Forward-only and bfloat16-only; unsupported inputs raise. + """ + kernel_name = "ulysses_flywheel" if use_ulysses else "flywheel" + if attention_mask is not None: + raise NotImplementedError(f"{kernel_name} does not support attention_mask.") + if is_causal: + raise NotImplementedError(f"{kernel_name} supports full (non-causal) attention only.") + if kv_heads is not None and kv_heads != heads: + raise NotImplementedError(f"{kernel_name} does not support GQA (got heads={heads}, kv_heads={kv_heads}).") + for name, tensor in (("query", query), ("key", key), ("value", value)): + if tensor.dtype != jnp.bfloat16: + raise TypeError(f"{kernel_name} requires bfloat16 inputs, got {name}.dtype={tensor.dtype}.") + + num_shards = mesh.shape[CONTEXT] if CONTEXT in mesh.shape else 1 + q_axis_names = nn.logical_to_mesh_axes(axis_names_q) + kv_axis_names = nn.logical_to_mesh_axes(axis_names_kv) + if use_ulysses: + _validate_implicit_ulysses_degree(ulysses_shards, num_shards, kernel_name) + if heads % num_shards != 0: + raise ValueError( + f"{kernel_name} requires the number of heads to be divisible by the context shard count, " + f"got heads={heads} and context_shards={num_shards}." + ) + elif num_shards > 1 and _mesh_axis_in_spec(kv_axis_names[2], CONTEXT): + raise ValueError( + f"{kernel_name} needs the full K/V sequence on every device, but the K/V sequence axis is " + f"sharded over '{CONTEXT}'. Use ulysses_flywheel for sequence-sharded K/V." + ) + + query, orig_q_seq_len = _reshape_data_for_flash(query, heads, num_shards) + num_shards_kv = num_shards if use_ulysses else 1 + key, orig_kv_seq_len = _reshape_data_for_flash(key, heads, num_shards_kv) + value, _ = _reshape_data_for_flash(value, heads, num_shards_kv) + + try: + # Optional dependency, imported only when the kernel is selected. + from flywheel_tpu import flash_attn_varlen_func # pylint: disable=import-outside-toplevel + except ImportError as e: + raise ImportError(f"{kernel_name} requires the flywheel-tpu package (pip install flywheel-tpu).") from e + + def local_attention(q, k, v): + if use_ulysses: + # [b, h, S/N, d] -> [b, h/N, S, d]: give up heads, gather the sequence. + q = jax.lax.all_to_all(q, axis_name=CONTEXT, split_axis=1, concat_axis=2, tiled=True) + k = jax.lax.all_to_all(k, axis_name=CONTEXT, split_axis=1, concat_axis=2, tiled=True) + v = jax.lax.all_to_all(v, axis_name=CONTEXT, split_axis=1, concat_axis=2, tiled=True) + q_len = orig_q_seq_len + else: + # Local query shard; its padding rows are computed and trimmed below. + q_len = q.shape[2] + b, h, s_q, d = q.shape + s_kv = k.shape[2] + out = flash_attn_varlen_func( + q.reshape(b * h, s_q, d), + k.reshape(b * h, s_kv, d), + v.reshape(b * h, s_kv, v.shape[-1]), + jnp.array([0, q_len], dtype=jnp.int32), + jnp.array([0, orig_kv_seq_len], dtype=jnp.int32), + max_seqlen_q=s_q, + max_seqlen_k=s_kv, + softmax_scale=float(scale), + causal=False, + ) + out = out.reshape(b, h, s_q, v.shape[-1]) + if use_ulysses: + # [b, h/N, S, d] -> [b, h, S/N, d] + out = jax.lax.all_to_all(out, axis_name=CONTEXT, split_axis=2, concat_axis=1, tiled=True) + return out + + sharded_attention = jax.shard_map( + local_attention, + mesh=mesh, + in_specs=(q_axis_names, kv_axis_names, kv_axis_names), + out_specs=q_axis_names, + check_vma=False, + ) + if use_ulysses: + x = _run_chunked_ulysses_attention(query, key, value, heads, num_shards, ulysses_attention_chunks, sharded_attention) + else: + x = sharded_attention(query, key, value) + x = x[:, :, :orig_q_seq_len, :] + return _reshape_heads_to_head_dim(x) + + # --------------------------------------------------------------------------- # Ulysses sequence-parallel attention # --------------------------------------------------------------------------- @@ -2733,6 +2845,44 @@ def flash_kernel(q, k, v, context): ) +@register_kernel("flywheel") +def flywheel_kernel(q, k, v, context): + return _flywheel_attention( + q, + k, + v, + context["heads"], + context["mesh"], + context["axis_names_q"], + context["axis_names_kv"], + # K may arrive already multiplied by the softmax scale (as attention="flash" accepts). + scale=1.0 if context.get("k_prescaled", False) else context["scale"], + attention_mask=context["attention_mask"], + is_causal=context.get("is_causal", False), + kv_heads=context.get("kv_heads", None), + ) + + +@register_kernel("ulysses_flywheel") +def ulysses_flywheel_kernel(q, k, v, context): + return _flywheel_attention( + q, + k, + v, + context["heads"], + context["mesh"], + context["axis_names_q"], + context["axis_names_kv"], + scale=1.0 if context.get("k_prescaled", False) else context["scale"], + use_ulysses=True, + attention_mask=context["attention_mask"], + is_causal=context.get("is_causal", False), + kv_heads=context.get("kv_heads", None), + ulysses_shards=context.get("ulysses_shards", -1), + ulysses_attention_chunks=context.get("ulysses_attention_chunks", 1), + ) + + @register_kernel("tokamax_flash") def tokamax_flash_kernel(q, k, v, context): return _tpu_flash_attention( @@ -2871,6 +3021,8 @@ def _apply_attention( if attention_kernel in [ "flash", "tokamax_flash", + "flywheel", + "ulysses_flywheel", "ulysses", "ulysses_custom", "ulysses_custom_fixed_m", @@ -3657,7 +3809,7 @@ def _wan_opt(name): self.wan_cross_attn_prescale_kv = _wan_opt("wan_cross_attn_prescale_kv") self.wan_rope_accum = _wan_opt("wan_rope_accum") - if attention_kernel in {"flash", "cudnn_flash_te"} and mesh is None: + if attention_kernel in {"flash", "cudnn_flash_te", *FLYWHEEL_KERNELS} and mesh is None: raise ValueError(f"The flash attention kernel requires a value for mesh, but mesh is {self.mesh}") self.dim_head = dim_head self.heads = heads @@ -3684,7 +3836,8 @@ def _wan_opt(name): "ulysses_custom_fixed_m_per_q_block", ) cross_attention_uses_local_kv = not is_self_attention and ( - cross_attention_remapped_to_flash or attention_kernel in ("flash", "tokamax_flash", "cudnn_flash_te") + cross_attention_remapped_to_flash + or attention_kernel in ("flash", "tokamax_flash", "cudnn_flash_te", *FLYWHEEL_KERNELS) ) if is_self_attention: axis_names_q = (BATCH, SELF_ATTN_HEAD, SELF_ATTN_Q_LENGTH, D_KV) @@ -3699,6 +3852,10 @@ def _wan_opt(name): ) if cross_attention_remapped_to_flash: attention_kernel = "tokamax_flash" + elif not is_self_attention and attention_kernel in FLYWHEEL_KERNELS: + # FlyWheel replaces self-attention only. Cross-attention (video queries x + # text keys, and the I2V image mask) stays on Splash. + attention_kernel = "flash" self.added_kv_proj_dim = added_kv_proj_dim # New for I2V self.image_seq_len = image_seq_len # New for I2V tpu_type = get_tpu_type() diff --git a/src/maxdiffusion/tests/flywheel_attention_test.py b/src/maxdiffusion/tests/flywheel_attention_test.py new file mode 100644 index 000000000..d16b6807c --- /dev/null +++ b/src/maxdiffusion/tests/flywheel_attention_test.py @@ -0,0 +1,268 @@ +""" +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +"""Tests for attention='flywheel' and attention='ulysses_flywheel'. + +The guard and wiring tests need neither flywheel-tpu nor a TPU: every +unsupported input must raise before the optional import. The numerical tests +need flywheel-tpu on a TPU the kernel supports (v6e or tpu7x) and compare +against an fp32 softmax reference; the multi-device ones also need 4 devices. +""" +# pylint: disable=protected-access +import importlib.util +import math +import unittest + +from flax import nnx +from flax.linen import partitioning as nn_partitioning +import jax +import jax.numpy as jnp +from jax.sharding import AbstractMesh, Mesh +import numpy as np + +from .. import common_types +from ..models import attention_flax +from ..tpu_utils import TpuType, get_tpu_type + +HAS_FLYWHEEL = importlib.util.find_spec("flywheel_tpu") is not None +FLYWHEEL_TPU = get_tpu_type() in (TpuType.TPU_V6_LITE, TpuType.TPU_7X) +_requires_flywheel = unittest.skipUnless(HAS_FLYWHEEL and FLYWHEEL_TPU, "needs flywheel-tpu on TPU v6e or tpu7x") +_requires_4_devices = unittest.skipIf(len(jax.devices()) < 4, "needs 4 devices for context parallelism") + +MESH_AXES = ("data", "fsdp", "context", "tensor") +Q_NAMES = (attention_flax.BATCH, attention_flax.SELF_ATTN_HEAD, attention_flax.SELF_ATTN_Q_LENGTH, attention_flax.D_KV) +KV_NAMES = (attention_flax.BATCH, attention_flax.SELF_ATTN_HEAD, attention_flax.SELF_ATTN_KV_LENGTH, attention_flax.D_KV) +# The rules pyconfig adds for attention='flywheel' (as for 'flash') and for 'ulysses*'. +FLASH_RULES = tuple(tuple(rule) for rule in common_types.SEQUENCE_PARALLEL_AXIS_RULES) + ((common_types.BATCH, "data"),) +ULYSSES_RULES = tuple(tuple(rule) for rule in common_types.ULYSSES_ATTENTION_AXIS_RULES) + ((common_types.BATCH, "data"),) + + +def _mesh(num_context): + devices = np.array(jax.devices()[:num_context]).reshape(1, 1, num_context, 1) + return Mesh(devices, MESH_AXES) + + +def _inputs(batch, seq, heads, head_dim, seed=0): + keys = jax.random.split(jax.random.PRNGKey(seed), 3) + shape = (batch, seq, heads * head_dim) + return tuple(jax.random.normal(k, shape, jnp.float32).astype(jnp.bfloat16) for k in keys) + + +def _reference(query, key, value, heads, scale): + """fp32 softmax attention on [B, S, H*D] inputs; returns [B, S, H*D].""" + b, s, hd = query.shape + q, k, v = (x.astype(jnp.float32).reshape(b, -1, heads, hd // heads).transpose(0, 2, 1, 3) for x in (query, key, value)) + probs = jax.nn.softmax(jnp.einsum("bhqd,bhkd->bhqk", q, k) * scale, axis=-1) + return jnp.einsum("bhqk,bhkd->bhqd", probs, v).transpose(0, 2, 1, 3).reshape(b, s, hd) + + +def _rel_err(out, ref): + return float(jnp.linalg.norm(out.astype(jnp.float32) - ref) / jnp.linalg.norm(ref)) + + +class FlywheelGuardTest(unittest.TestCase): + """Unsupported inputs raise before flywheel_tpu is imported, so these run anywhere.""" + + def setUp(self): + self.x = jnp.zeros((1, 256, 2 * 128), jnp.bfloat16) + + def _call(self, mesh=None, rules=FLASH_RULES, x=None, **kwargs): + mesh = mesh if mesh is not None else AbstractMesh((1, 1, 1, 1), MESH_AXES) + x = self.x if x is None else x + with nn_partitioning.axis_rules(rules): + return attention_flax._flywheel_attention( + x, x, x, heads=2, mesh=mesh, axis_names_q=Q_NAMES, axis_names_kv=KV_NAMES, scale=1.0, **kwargs + ) + + def test_kernels_are_registered(self): + for name in attention_flax.FLYWHEEL_KERNELS: + self.assertIn(name, attention_flax.KERNEL_REGISTRY) + + def test_attention_mask_is_rejected(self): + for use_ulysses in (False, True): + with self.assertRaisesRegex(NotImplementedError, "attention_mask"): + self._call(attention_mask=jnp.ones((1, 256), jnp.bool_), use_ulysses=use_ulysses) + + def test_causal_is_rejected(self): + with self.assertRaisesRegex(NotImplementedError, "non-causal"): + self._call(is_causal=True) + + def test_gqa_is_rejected(self): + with self.assertRaisesRegex(NotImplementedError, "GQA"): + self._call(kv_heads=1) + + def test_non_bfloat16_is_rejected(self): + with self.assertRaisesRegex(TypeError, "bfloat16"): + self._call(x=self.x.astype(jnp.float32)) + + def test_sequence_sharded_kv_is_rejected_without_ulysses(self): + with self.assertRaisesRegex(ValueError, "ulysses_flywheel"): + self._call(mesh=AbstractMesh((1, 1, 2, 1), MESH_AXES), rules=ULYSSES_RULES) + + def test_ulysses_rejects_indivisible_heads(self): + with self.assertRaisesRegex(ValueError, "divisible"): + self._call(mesh=AbstractMesh((1, 1, 3, 1), MESH_AXES), rules=ULYSSES_RULES, use_ulysses=True) + + def test_ulysses_rejects_mismatched_ulysses_shards(self): + with self.assertRaisesRegex(ValueError, "ulysses_shards=4"): + self._call(mesh=AbstractMesh((1, 1, 2, 1), MESH_AXES), rules=ULYSSES_RULES, use_ulysses=True, ulysses_shards=4) + + +class FlywheelWanWiringTest(unittest.TestCase): + """FlaxWanAttention runs self-attention on FlyWheel and keeps cross-attention on Splash.""" + + def _build(self, attention_kernel, is_self_attention): + with _mesh(1), nn_partitioning.axis_rules(FLASH_RULES): + return attention_flax.FlaxWanAttention( + rngs=nnx.Rngs(0), + query_dim=256, + heads=2, + dim_head=128, + attention_kernel=attention_kernel, + mesh=_mesh(1), + is_self_attention=is_self_attention, + ) + + def test_self_attention_uses_flywheel_and_cross_attention_uses_flash(self): + for kernel in attention_flax.FLYWHEEL_KERNELS: + self.assertEqual(self._build(kernel, is_self_attention=True).attention_op.attention_kernel, kernel) + self.assertEqual(self._build(kernel, is_self_attention=False).attention_op.attention_kernel, "flash") + + def test_mesh_is_required(self): + with self.assertRaises(ValueError): + attention_flax.FlaxWanAttention( + rngs=nnx.Rngs(0), query_dim=256, heads=2, dim_head=128, attention_kernel="flywheel", mesh=None + ) + + +@_requires_flywheel +class FlywheelNumericsTest(unittest.TestCase): + """Sharded FlyWheel attention against an fp32 reference on the same bf16 inputs.""" + + # bf16 inputs against an fp32 reference: the observed error is ~3e-3. + TOL = 1e-2 + + def _check(self, *, num_context=1, use_ulysses=False, chunks=1, batch=2, seq=1024, heads=4, head_dim=128): + """Runs the sharded backend on random bf16 inputs and compares with the fp32 reference.""" + query, key, value = _inputs(batch, seq, heads, head_dim) + scale = 1.0 / math.sqrt(head_dim) + mesh = _mesh(num_context) + with mesh, nn_partitioning.axis_rules(ULYSSES_RULES if use_ulysses else FLASH_RULES): + out = jax.jit( + lambda q, k, v: attention_flax._flywheel_attention( + q, + k, + v, + heads=heads, + mesh=mesh, + axis_names_q=Q_NAMES, + axis_names_kv=KV_NAMES, + scale=scale, + use_ulysses=use_ulysses, + ulysses_attention_chunks=chunks, + ) + )(query, key, value) + ref = _reference(query, key, value, heads, scale) + self.assertEqual(out.shape, ref.shape) + self.assertEqual(out.dtype, jnp.bfloat16) + self.assertLess(_rel_err(out, ref), self.TOL) + + def test_single_device(self): + self._check() + + def test_single_device_ulysses(self): + self._check(use_ulysses=True) + + def test_unaligned_length_does_not_attend_padding(self): + # 600 is not a multiple of 128: the kernel must not attend the rows it pads to. + self._check(seq=600) + + def test_single_batch_row(self): + self._check(batch=1, heads=3) + + def test_k_prescaled_matches_scale_argument(self): + query, key, value = _inputs(2, 1024, 4, 128) + scale = 1.0 / math.sqrt(128) + mesh = _mesh(1) + outs = [] + with mesh, nn_partitioning.axis_rules(FLASH_RULES): + for k, prescaled in ((key, False), ((key.astype(jnp.float32) * scale).astype(jnp.bfloat16), True)): + context = { + "heads": 4, + "mesh": mesh, + "axis_names_q": Q_NAMES, + "axis_names_kv": KV_NAMES, + "scale": scale, + "k_prescaled": prescaled, + "attention_mask": None, + } + outs.append(attention_flax.KERNEL_REGISTRY["flywheel"](query, k, value, context)) + self.assertLess(_rel_err(outs[1], outs[0].astype(jnp.float32)), self.TOL) + + def test_gradient_is_not_supported(self): + query, key, value = _inputs(1, 256, 2, 128) + mesh = _mesh(1) + + def loss(q): + out = attention_flax._flywheel_attention( + q, key, value, heads=2, mesh=mesh, axis_names_q=Q_NAMES, axis_names_kv=KV_NAMES, scale=1.0 + ) + return out.astype(jnp.float32).sum() + + with mesh, nn_partitioning.axis_rules(FLASH_RULES): + with self.assertRaises(NotImplementedError): + jax.grad(loss)(query) + + def test_wan_self_attention_matches_flash(self): + """Same FlaxWanAttention weights and inputs, attention='flash' vs attention='flywheel'.""" + heads, head_dim, seq = 4, 128, 1560 + hidden = jax.random.normal(jax.random.PRNGKey(1), (2, seq, heads * head_dim), jnp.float32).astype(jnp.bfloat16) + angles = jax.random.uniform(jax.random.PRNGKey(2), (1, 1, seq, head_dim // 2), jnp.float32, 0.0, 2 * math.pi) + rotary_emb = jax.lax.complex(jnp.cos(angles), jnp.sin(angles)) + mesh = _mesh(1) + outs = {} + with mesh, nn_partitioning.axis_rules(FLASH_RULES): + for kernel in ("flash", "flywheel"): + attn = attention_flax.FlaxWanAttention( + rngs=nnx.Rngs(0), + query_dim=heads * head_dim, + heads=heads, + dim_head=head_dim, + attention_kernel=kernel, + mesh=mesh, + flash_min_seq_length=0, + dtype=jnp.bfloat16, + weights_dtype=jnp.bfloat16, + ) + outs[kernel] = nnx.jit(lambda m, h: m(hidden_states=h, rotary_emb=rotary_emb))(attn, hidden) + self.assertLess(_rel_err(outs["flywheel"], outs["flash"].astype(jnp.float32)), self.TOL) + + @_requires_4_devices + def test_context_parallel(self): + # q is sequence-sharded 4 ways and K/V replicated; 1022 % 4 != 0 exercises the context padding. + for seq in (1024, 1022): + with self.subTest(seq=seq): + self._check(num_context=4, seq=seq) + + @_requires_4_devices + def test_context_parallel_ulysses(self): + for seq, chunks in ((1024, 1), (1022, 1), (1024, 2)): + with self.subTest(seq=seq, chunks=chunks): + self._check(num_context=4, use_ulysses=True, chunks=chunks, heads=8, seq=seq) + + +if __name__ == "__main__": + unittest.main()