V3: unified multi-backend video generation framework

Add support for 5 video generation backends on consumer GPUs (6-24GB):
- Wan 2.1 (1.3B/14B) — best quality/VRAM ratio
- HunyuanVideo (8.3B) — high quality with NF4 quantization
- CogVideoX (2B/5B) — lightweight entry point
- LTX-Video — real-time 8-step inference
- AnimateDiff legacy — full backward compatibility

Core infrastructure:
- Smart VRAM manager: auto-detects GPU, recommends optimal config
- Unified base pipeline: load/generate/save interface for all backends
- Quantization layer: BitsAndBytes NF4/INT8, torchao FP8
- torch.compile support for Ampere+ GPUs

CLI: `animatediff --backend wan --prompt "..." --quality high`
  --backend {auto|wan|hunyuan|cogvideo|ltx|animatediff}
  --quality {draft|standard|high|max} presets
  --auto mode picks best backend for available hardware
  All existing --pipeline flags preserved for backward compat

WebUI: new V3 Multi-Backend tab alongside legacy AnimateDiff tab

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: charliewwdev

animatediff/backends/__init__.py

@@ -0,0 +1,35 @@
+"""
+Video generation backends — unified access to multiple model families.
+
+Available backends:
+- wan: Wan 2.1 (Alibaba, 1.3B-14B, best quality-to-VRAM ratio)
+- hunyuan: HunyuanVideo (Tencent, 8.3B, high quality)
+- cogvideo: CogVideoX (THU, 2B-5B, lightest)
+- ltx: LTX-Video (Lightricks, real-time capable)
+- animatediff: AnimateDiff legacy (SD1.5/SDXL/Lightning)
+"""
+
+from typing import Dict, Type
+
+BACKEND_REGISTRY: Dict[str, str] = {
+    "wan": "animatediff.backends.wan.WanBackend",
+    "hunyuan": "animatediff.backends.hunyuan.HunyuanBackend",
+    "cogvideo": "animatediff.backends.cogvideo.CogVideoBackend",
+    "ltx": "animatediff.backends.ltx.LTXBackend",
+    "animatediff": "animatediff.backends.animatediff_legacy.AnimateDiffBackend",
+}
+
+
+def get_backend(name: str):
+    """Lazily import and return a backend class by name."""
+    if name not in BACKEND_REGISTRY:
+        raise ValueError(f"Unknown backend: {name}. Available: {list(BACKEND_REGISTRY.keys())}")
+
+    module_path, class_name = BACKEND_REGISTRY[name].rsplit(".", 1)
+    import importlib
+    module = importlib.import_module(module_path)
+    return getattr(module, class_name)
+
+
+def list_backends():
+    return list(BACKEND_REGISTRY.keys())

animatediff/backends/animatediff_legacy.py

@@ -0,0 +1,123 @@
+"""
+AnimateDiff Legacy Backend — wraps existing V2/SDXL/Lightning/Legacy pipelines.
+
+Provides backward compatibility with all existing AnimateDiff features:
+- SD1.5 text-to-video with motion modules
+- SDXL text-to-video
+- Lightning ultra-fast inference (1-8 steps)
+- SparseCtrl (legacy pipeline only)
+- FreeInit, FreeNoise, IP-Adapter, Prompt Travel (V2)
+"""
+
+import logging
+from typing import Optional
+
+import torch
+from PIL import Image
+
+from animatediff.core.base_pipeline import BasePipeline, VideoOutput
+
+logger = logging.getLogger(__name__)
+
+
+class AnimateDiffBackend(BasePipeline):
+    """Wraps existing AnimateDiff pipelines (V2/SDXL/Lightning/Legacy)."""
+
+    backend_name = "animatediff"
+
+    def __init__(self, pipe, pipeline_type: str = "v2"):
+        self.pipe = pipe
+        self.pipeline_type = pipeline_type
+
+    @classmethod
+    def load(
+        cls,
+        model_path: Optional[str] = None,
+        torch_dtype: torch.dtype = torch.float16,
+        device: str = "cuda",
+        quantization: str = "none",
+        offload_strategy: str = "none",
+        enable_vae_slicing: bool = True,
+        enable_vae_tiling: bool = False,
+        pipeline_type: str = "v2",
+        scheduler: str = "ddim",
+        motion_adapter: Optional[str] = None,
+        lightning_steps: int = 4,
+        **kwargs,
+    ) -> "AnimateDiffBackend":
+        if model_path is None:
+            model_path = "runwayml/stable-diffusion-v1-5"
+
+        if pipeline_type == "sdxl":
+            from animatediff.pipelines.pipeline_sdxl import AnimateDiffSDXL
+            pipe = AnimateDiffSDXL.from_pretrained(
+                model_path=model_path or "stabilityai/stable-diffusion-xl-base-1.0",
+                motion_adapter_path=motion_adapter or "guoyww/animatediff-motion-adapter-sdxl-beta",
+                torch_dtype=torch_dtype,
+                device=device,
+                scheduler=scheduler,
+            )
+        elif pipeline_type == "lightning":
+            from animatediff.pipelines.pipeline_lightning import AnimateDiffLightning
+            pipe = AnimateDiffLightning.from_pretrained(
+                model_path=model_path or "emilianJR/epiCRealism",
+                num_steps=lightning_steps,
+                torch_dtype=torch_dtype,
+                device=device,
+            )
+        else:
+            from animatediff.pipelines.pipeline_v2 import AnimateDiffV2Pipeline
+            pipe = AnimateDiffV2Pipeline.from_pretrained(
+                model_path=model_path,
+                motion_adapter_path=motion_adapter or "guoyww/animatediff-motion-adapter-v1-5-3",
+                torch_dtype=torch_dtype,
+                device=device,
+                scheduler=scheduler,
+                enable_vae_slicing=enable_vae_slicing,
+                enable_vae_tiling=enable_vae_tiling,
+            )
+
+        return cls(pipe, pipeline_type=pipeline_type)
+
+    @torch.no_grad()
+    def generate(
+        self,
+        prompt: str,
+        negative_prompt: str = "bad quality, worst quality",
+        width: int = 512,
+        height: int = 512,
+        num_frames: int = 16,
+        num_inference_steps: int = 25,
+        guidance_scale: float = 7.5,
+        seed: int = -1,
+        image: Optional[Image.Image] = None,
+        **kwargs,
+    ) -> VideoOutput:
+        output = self.pipe.generate(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            num_frames=num_frames,
+            height=height,
+            width=width,
+            num_inference_steps=num_inference_steps,
+            guidance_scale=guidance_scale,
+            seed=seed,
+            **kwargs,
+        )
+        frames = output.frames[0]
+
+        return VideoOutput(
+            frames=frames,
+            seed=seed,
+            backend=self.backend_name,
+            metadata={"pipeline_type": self.pipeline_type},
+        )
+
+    def save(self, output: VideoOutput, path: str, fps: int = 8):
+        """Use existing pipeline save or default."""
+        if hasattr(self.pipe, "save"):
+            # The V2/SDXL/Lightning pipelines have their own save method
+            # but they expect their native output format, not VideoOutput.
+            # Use base class save instead.
+            pass
+        super().save(output, path, fps=fps)

animatediff/backends/cogvideo.py

@@ -0,0 +1,113 @@
+"""
+CogVideoX Backend — wraps diffusers CogVideoXPipeline.
+
+Model variants:
+- THUDM/CogVideoX-2b  (6 GB VRAM with FP8, lightest option)
+- THUDM/CogVideoX-5b  (12+ GB VRAM, better quality)
+
+Features:
+- Text-to-video with 3D causal VAE
+- Adaptive LayerNorm for text-video alignment
+- 3D full attention for motion capture
+- LoRA support built-in
+"""
+
+import logging
+from typing import Optional
+
+import torch
+from PIL import Image
+
+from animatediff.core.base_pipeline import BasePipeline, VideoOutput
+from animatediff.core.quantization import get_quantization_config
+
+logger = logging.getLogger(__name__)
+
+COGVIDEO_MODELS = {
+    "2B": "THUDM/CogVideoX-2b",
+    "5B": "THUDM/CogVideoX-5b",
+}
+
+
+class CogVideoBackend(BasePipeline):
+    backend_name = "cogvideo"
+
+    def __init__(self, pipe, model_variant: str = "2B"):
+        self.pipe = pipe
+        self.model_variant = model_variant
+
+    @classmethod
+    def load(
+        cls,
+        model_path: Optional[str] = None,
+        torch_dtype: torch.dtype = torch.float16,
+        device: str = "cuda",
+        quantization: str = "none",
+        offload_strategy: str = "none",
+        enable_vae_slicing: bool = True,
+        enable_vae_tiling: bool = False,
+        model_variant: str = "2B",
+        **kwargs,
+    ) -> "CogVideoBackend":
+        from diffusers import CogVideoXPipeline
+
+        if model_path is None:
+            model_path = COGVIDEO_MODELS.get(model_variant, COGVIDEO_MODELS["2B"])
+
+        logger.info(f"Loading CogVideoX-{model_variant} from {model_path} (dtype={torch_dtype}, quant={quantization})")
+
+        quant_config = get_quantization_config(quantization, components=["transformer"])
+
+        load_kwargs = dict(torch_dtype=torch_dtype)
+        if quant_config is not None:
+            load_kwargs["quantization_config"] = quant_config
+
+        pipe = CogVideoXPipeline.from_pretrained(model_path, **load_kwargs)
+
+        instance = cls(pipe, model_variant=model_variant)
+
+        if offload_strategy != "none":
+            instance._apply_offloading(pipe, offload_strategy)
+        else:
+            pipe.to(device)
+
+        instance._apply_vae_opts(pipe, slicing=enable_vae_slicing, tiling=enable_vae_tiling)
+
+        return instance
+
+    @torch.no_grad()
+    def generate(
+        self,
+        prompt: str,
+        negative_prompt: str = "",
+        width: int = 720,
+        height: int = 480,
+        num_frames: int = 49,
+        num_inference_steps: int = 50,
+        guidance_scale: float = 6.0,
+        seed: int = -1,
+        image: Optional[Image.Image] = None,
+        **kwargs,
+    ) -> VideoOutput:
+        gen_device = "cpu" if self.pipe.device.type == "cpu" else self.pipe.device
+        generator = self._make_generator(seed, gen_device)
+
+        output = self.pipe(
+            prompt=prompt,
+            negative_prompt=negative_prompt or None,
+            width=width,
+            height=height,
+            num_frames=num_frames,
+            num_inference_steps=num_inference_steps,
+            guidance_scale=guidance_scale,
+            generator=generator,
+            output_type="pil",
+        )
+        frames = output.frames[0]
+
+        return VideoOutput(
+            frames=frames,
+            seed=seed,
+            backend=self.backend_name,
+            metadata={"model_variant": self.model_variant},
+        )

animatediff/backends/hunyuan.py

@@ -0,0 +1,108 @@
+"""
+HunyuanVideo Backend — wraps diffusers HunyuanVideoPipeline.
+
+Model: Tencent-Hunyuan/HunyuanVideo (8.3B parameters)
+
+Features:
+- High-quality text-to-video generation
+- Dual-stream to single-stream transformer architecture
+- 3D causal VAE for temporal coherence
+- BitsAndBytes NF4 quantization (runs on 8GB VRAM)
+"""
+
+import logging
+from typing import Optional
+
+import torch
+from PIL import Image
+
+from animatediff.core.base_pipeline import BasePipeline, VideoOutput
+from animatediff.core.quantization import get_quantization_config
+
+logger = logging.getLogger(__name__)
+
+HUNYUAN_MODELS = {
+    "default": "tencent/HunyuanVideo",
+}
+
+
+class HunyuanBackend(BasePipeline):
+    backend_name = "hunyuan"
+
+    def __init__(self, pipe, model_variant: str = "default"):
+        self.pipe = pipe
+        self.model_variant = model_variant
+
+    @classmethod
+    def load(
+        cls,
+        model_path: Optional[str] = None,
+        torch_dtype: torch.dtype = torch.float16,
+        device: str = "cuda",
+        quantization: str = "none",
+        offload_strategy: str = "model_cpu",
+        enable_vae_slicing: bool = True,
+        enable_vae_tiling: bool = True,
+        model_variant: str = "default",
+        **kwargs,
+    ) -> "HunyuanBackend":
+        from diffusers import HunyuanVideoPipeline
+
+        if model_path is None:
+            model_path = HUNYUAN_MODELS.get(model_variant, HUNYUAN_MODELS["default"])
+
+        logger.info(f"Loading HunyuanVideo from {model_path} (dtype={torch_dtype}, quant={quantization})")
+
+        quant_config = get_quantization_config(quantization, components=["transformer"])
+
+        load_kwargs = dict(torch_dtype=torch_dtype)
+        if quant_config is not None:
+            load_kwargs["quantization_config"] = quant_config
+
+        pipe = HunyuanVideoPipeline.from_pretrained(model_path, **load_kwargs)
+
+        instance = cls(pipe, model_variant=model_variant)
+
+        if offload_strategy != "none":
+            instance._apply_offloading(pipe, offload_strategy)
+        else:
+            pipe.to(device)
+
+        instance._apply_vae_opts(pipe, slicing=enable_vae_slicing, tiling=enable_vae_tiling)
+
+        return instance
+
+    @torch.no_grad()
+    def generate(
+        self,
+        prompt: str,
+        negative_prompt: str = "",
+        width: int = 720,
+        height: int = 480,
+        num_frames: int = 33,
+        num_inference_steps: int = 30,
+        guidance_scale: float = 6.0,
+        seed: int = -1,
+        image: Optional[Image.Image] = None,
+        **kwargs,
+    ) -> VideoOutput:
+        gen_device = "cpu" if self.pipe.device.type == "cpu" else self.pipe.device
+        generator = self._make_generator(seed, gen_device)
+
+        output = self.pipe(
+            prompt=prompt,
+            width=width,
+            height=height,
+            num_frames=num_frames,
+            num_inference_steps=num_inference_steps,
+            guidance_scale=guidance_scale,
+            generator=generator,
+            output_type="pil",
+        )
+        frames = output.frames[0]
+
+        return VideoOutput(
+            frames=frames,
+            seed=seed,
+            backend=self.backend_name,
+        )