From 5731088a9681af0ce059e728cce88c316697eba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Fri, 21 Aug 2026 14:28:39 +0000 Subject: [PATCH 1/5] Raise a clear error when FSDP is enabled on a mesh with no shard dimension `ParallelismConfig.fsdp_dim_names` always asks for `dp_shard_cp`, but that joint dimension is only flattened into the device mesh when `dp_shard` or `cp` is enabled. Launching an already-parallelized model -- e.g. one loaded with `DistributedConfig(tp_size=world_size)` or `enable_expert_parallel=True` -- under an FSDP config therefore slices a tp-only mesh by a name it does not contain, and the user sees a bare `KeyError` from `torch.distributed.device_mesh`, several frames below any code they wrote. Say what is actually wrong instead. --- src/accelerate/utils/fsdp_utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/accelerate/utils/fsdp_utils.py b/src/accelerate/utils/fsdp_utils.py index df4df42ad84..71255429942 100644 --- a/src/accelerate/utils/fsdp_utils.py +++ b/src/accelerate/utils/fsdp_utils.py @@ -761,6 +761,19 @@ def fsdp2_prepare_model(accelerator, model: torch.nn.Module) -> torch.nn.Module: fsdp2_plugin.set_auto_wrap_policy(model) mesh = getattr(accelerator, "torch_device_mesh", None) + pc = accelerator.parallelism_config + + # `fsdp_dim_names` always asks for `dp_shard_cp`, but that joint dimension is only flattened into the mesh when + # `dp_shard` or `cp` is enabled. Without either, slicing the mesh below raises a `KeyError` from `device_mesh`. + if mesh is not None and not pc.dp_shard_enabled and not pc.cp_enabled: + raise ValueError( + f"FSDP is enabled but the device mesh is {tuple(mesh.mesh_dim_names)}, which has no dimension for FSDP " + "to shard across (both `dp_shard_size` and `cp_size` are 1). This usually means a model that is already " + "parallelized another way -- e.g. loaded with `DistributedConfig(tp_size=N)` or " + "`enable_expert_parallel=True`, which makes the whole world size tensor/expert parallel -- was launched " + "under an FSDP config. Either launch it without the FSDP config, or leave ranks for FSDP to use by " + "lowering `tp_size`." + ) fsdp2_kwargs = { "reshard_after_forward": fsdp2_plugin.reshard_after_forward, From a9e2e6b5fce8a4fda811551d8e8f3a700731cc16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Sat, 22 Aug 2026 07:51:03 +0000 Subject: [PATCH 2/5] is_peft_model: trust the imported peft, not just distribution metadata is_peft_available() checks importlib metadata, so a peft provided only through PYTHONPATH (source checkout, shipped environment) makes is_peft_model() return False even while a PeftModel instance is right there in the model tree. Callers then treat the wrapper as a plain model; in TRL this patched a fused-loss forward onto the PeftModel, where transformers' base_model property resolves to peft's LoraModel and training crashes with an unrelated-looking AttributeError. If peft is already in sys.modules, isinstance against it directly; fall back to the metadata check only to trigger the first import. --- src/accelerate/utils/modeling.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/accelerate/utils/modeling.py b/src/accelerate/utils/modeling.py index 2ac8be12fed..bfe6dd9fe8a 100644 --- a/src/accelerate/utils/modeling.py +++ b/src/accelerate/utils/modeling.py @@ -20,6 +20,7 @@ import os import re import shutil +import sys import tempfile import warnings from collections import OrderedDict, defaultdict @@ -73,10 +74,15 @@ def is_peft_model(model): from .other import extract_model_from_parallel - if is_peft_available(): - from peft import PeftModel - - return is_peft_available() and isinstance(extract_model_from_parallel(model), PeftModel) + # `is_peft_available()` checks distribution metadata, which misses a peft provided only through + # `PYTHONPATH` / `sys.path` (no dist-info). If peft is already imported, trust the import: a + # `PeftModel` instance can only exist if its module does. + peft = sys.modules.get("peft") + if peft is None and is_peft_available(): + import peft + if peft is None: + return False + return isinstance(extract_model_from_parallel(model), peft.PeftModel) def check_device_same(first_device, second_device): From da22b48d656b610c60c69512223d55bb56a7cf6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Sat, 22 Aug 2026 09:32:05 +0000 Subject: [PATCH 3/5] prepare_model: don't move DTensor-sharded models to the device A model sharded at load time (e.g. transformers DistributedConfig with FSDP2, optionally CPU-offloaded) manages its own parameter placement, and .to() on FSDP2-managed parameters raises RuntimeError: _apply(): Couldn't swap FSDPLinear.weight. model_has_dtensor already exists and is checked a few lines below for the DDP case; use it to skip the device move too. --- src/accelerate/accelerator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py index 3b8829fabfb..559f7ffe304 100755 --- a/src/accelerate/accelerator.py +++ b/src/accelerate/accelerator.py @@ -1871,7 +1871,10 @@ def prepare_model( "You can't train a model that has been loaded in 8-bit or 4-bit precision with CPU or disk offload. " "If you want train the 8-bit or 4-bit model in CPU, please install bitsandbytes with multi-backend, see https://huggingface.co/docs/bitsandbytes/main/en/installation#multi-backend" ) - elif device_placement and not self.verify_device_map(model): + elif device_placement and not self.verify_device_map(model) and not model_has_dtensor(model): + # A DTensor-sharded model (e.g. sharded at load by transformers' DistributedConfig) + # manages its own placement; `.to()` on FSDP2-managed or CPU-offloaded parameters + # raises `_apply(): Couldn't swap ...`. model = model.to(self.device) if not evaluation_mode: if self.multi_device and not (self.parallelism_config and self.parallelism_config.tp_enabled): From c3f972d763fa71a17c6116c69fb91cad241c4e4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Sun, 23 Aug 2026 19:57:19 +0000 Subject: [PATCH 4/5] Revert "is_peft_model: trust the imported peft, not just distribution metadata" This reverts commit a9e2e6b5fce8a4fda811551d8e8f3a700731cc16. --- src/accelerate/utils/modeling.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/accelerate/utils/modeling.py b/src/accelerate/utils/modeling.py index bfe6dd9fe8a..2ac8be12fed 100644 --- a/src/accelerate/utils/modeling.py +++ b/src/accelerate/utils/modeling.py @@ -20,7 +20,6 @@ import os import re import shutil -import sys import tempfile import warnings from collections import OrderedDict, defaultdict @@ -74,15 +73,10 @@ def is_peft_model(model): from .other import extract_model_from_parallel - # `is_peft_available()` checks distribution metadata, which misses a peft provided only through - # `PYTHONPATH` / `sys.path` (no dist-info). If peft is already imported, trust the import: a - # `PeftModel` instance can only exist if its module does. - peft = sys.modules.get("peft") - if peft is None and is_peft_available(): - import peft - if peft is None: - return False - return isinstance(extract_model_from_parallel(model), peft.PeftModel) + if is_peft_available(): + from peft import PeftModel + + return is_peft_available() and isinstance(extract_model_from_parallel(model), PeftModel) def check_device_same(first_device, second_device): From 338c19a513efee26bf60b955f7b25db50ec126ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= Date: Sun, 23 Aug 2026 20:16:32 +0000 Subject: [PATCH 5/5] Wrap the comment into one line --- src/accelerate/accelerator.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/accelerate/accelerator.py b/src/accelerate/accelerator.py index 559f7ffe304..b2ddc845c45 100755 --- a/src/accelerate/accelerator.py +++ b/src/accelerate/accelerator.py @@ -1872,9 +1872,7 @@ def prepare_model( "If you want train the 8-bit or 4-bit model in CPU, please install bitsandbytes with multi-backend, see https://huggingface.co/docs/bitsandbytes/main/en/installation#multi-backend" ) elif device_placement and not self.verify_device_map(model) and not model_has_dtensor(model): - # A DTensor-sharded model (e.g. sharded at load by transformers' DistributedConfig) - # manages its own placement; `.to()` on FSDP2-managed or CPU-offloaded parameters - # raises `_apply(): Couldn't swap ...`. + # DTensor-sharded models manage their own placement; `.to()` on FSDP2-managed or CPU-offloaded params raises `_apply(): Couldn't swap ...` model = model.to(self.device) if not evaluation_mode: if self.multi_device and not (self.parallelism_config and self.parallelism_config.tp_enabled):