diff --git a/.dockerignore b/.dockerignore index c042738..e1171a3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,6 +16,7 @@ dist .env.* **/ray-auth-token remote_exec.sh +AGENTS.override.md **/*.safetensors **/*.pt container-cache diff --git a/aspec/_environment.py b/aspec/_environment.py index 696710f..8c1a5ad 100644 --- a/aspec/_environment.py +++ b/aspec/_environment.py @@ -92,7 +92,15 @@ def command_environment(): raise ValueError("ASPEC_CUDA_COMPAT_DIR must contain libcuda.so.1.") paths.append(str(path)) purelib = Path(sysconfig.get_paths()["purelib"]) + cudnn = purelib / "nvidia/cudnn" + if not env.get("CUDNN_HOME") and (cudnn / "lib/libcudnn.so.9").is_file(): + # TE searches CUDA_HOME and the host toolkit before LD_LIBRARY_PATH. + # Bind its explicit search to this interpreter's cuDNN wheel. + env["CUDNN_HOME"] = str(cudnn) + env["ASPEC_AUTO_CUDNN_HOME"] = str(cudnn) toolkit = purelib / "nvidia/cu13" + if (toolkit / "lib/libcudart.so.13").is_file(): + env.setdefault("CUDNN_FRONTEND_CUDART_LIB_NAME", "libcudart.so.13") if not env.get("CUDA_HOME") and not env.get("CUDA_PATH") and (toolkit / "bin/nvcc").is_file(): view = _toolkit_view(toolkit, env) env["CUDA_HOME"] = str(view) @@ -101,7 +109,8 @@ def command_environment(): previous = env.get("LIBRARY_PATH", "") env["LIBRARY_PATH"] = str(view / "lib64") + (os.pathsep + previous if previous else "") paths.append(str(view / "lib64")) - paths.extend(str(p) for p in (purelib / "nvidia/cuda_runtime/lib", purelib / "torch/lib") + paths.extend(str(p) for p in (purelib / "nvidia/cuda_runtime/lib", purelib / "torch/lib", + purelib / "nvidia/cudnn/lib", purelib / "nvidia/nccl/lib") if p.is_dir()) if (toolkit / "lib").is_dir(): paths.append(str(toolkit / "lib")) diff --git a/aspec/_target_worker.py b/aspec/_target_worker.py index 5655ad4..b4068dc 100644 --- a/aspec/_target_worker.py +++ b/aspec/_target_worker.py @@ -13,6 +13,9 @@ def main(): automatic_toolkit = os.environ.pop("ASPEC_AUTO_CUDA_HOME", None) if automatic_toolkit and os.environ.get("CUDA_HOME") == automatic_toolkit: os.environ.pop("CUDA_HOME", None) + automatic_cudnn = os.environ.pop("ASPEC_AUTO_CUDNN_HOME", None) + if automatic_cudnn and os.environ.get("CUDNN_HOME") == automatic_cudnn: + os.environ.pop("CUDNN_HOME", None) os.environ["PATH"] = str(Path(sys.executable).parent) + os.pathsep + os.environ.get("PATH", "") if cache := os.environ.get("ASPEC_CACHE_HOME"): if Path(cache).name != "target": diff --git a/aspec/cli.py b/aspec/cli.py index fec66ca..49b923a 100644 --- a/aspec/cli.py +++ b/aspec/cli.py @@ -48,7 +48,10 @@ def main(): else: module = "aspec.train_entry" help_requested = arguments in (["--help"], ["-h"]) - if module == "aspec.inference.service" and not help_requested and not is_target_environment(): + megatron_target = ("--backend=megatron" in arguments or any( + a == "--backend" and b == "megatron" for a, b in zip(arguments, arguments[1:]) + )) + if module == "aspec.inference.service" and not help_requested and not is_target_environment() and not megatron_target: raise SystemExit("Start aspec target in the Target environment (pip install ling-aspec-target)") try: execute([sys.executable, "-m", module, *arguments]) diff --git a/aspec/commands/_checkpoint_audit.py b/aspec/commands/_checkpoint_audit.py index 8b616ae..10a3c75 100644 --- a/aspec/commands/_checkpoint_audit.py +++ b/aspec/commands/_checkpoint_audit.py @@ -21,19 +21,19 @@ import math -def cpu_values(value): +def cpu_values(value, *, full_tensors=False): import torch if isinstance(value, torch.Tensor): if hasattr(value, "to_local"): - value = value.to_local() + value = value.full_tensor() if full_tensors else value.to_local() return value.detach().cpu().clone() if isinstance(value, dict): - return {key: cpu_values(item) for key, item in value.items()} + return {key: cpu_values(item, full_tensors=full_tensors) for key, item in value.items()} if isinstance(value, list): - return [cpu_values(item) for item in value] + return [cpu_values(item, full_tensors=full_tensors) for item in value] if isinstance(value, tuple): - return tuple(cpu_values(item) for item in value) + return tuple(cpu_values(item, full_tensors=full_tensors) for item in value) return value @@ -80,8 +80,12 @@ def compare_checkpoint(trainer, directory): candidate_model = ModelState(trainer.model).state_dict() candidate_optimizer = OptimizerState(trainer.model, trainer.optimizer).state_dict() - original_model = cpu_values(candidate_model) - original_optimizer = cpu_values(candidate_optimizer) + # Core compute parameters are ordinary local tensors, while its DCP state + # describes logical TP tensors. Read full independent CPU references rather + # than treating a local shard as the checkpoint's complete destination. + logical_tp = getattr(trainer, "execution_backend", "fsdp") == "megatron" + original_model = cpu_values(candidate_model, full_tensors=logical_tp) + original_optimizer = cpu_values(candidate_optimizer, full_tensors=logical_tp) dcp.load({"model_state": original_model}, checkpoint_id=str(directory / "model")) dcp.load({"optim_state": original_optimizer}, checkpoint_id=str(directory / "optimizer")) statistics = { @@ -91,6 +95,8 @@ def compare_checkpoint(trainer, directory): worst_parameters = {key: {"relative_l2": 0, "name": None} for key in statistics} def compare(key, name, reference, actual): + if logical_tp and hasattr(actual, "full_tensor"): + actual = actual.full_tensor() reference = reference.reshape(-1) actual = actual.reshape(-1) if reference.shape != actual.shape: @@ -127,16 +133,25 @@ def compare(key, name, reference, actual): } +def backend_rng_identity(state): + return {"backend": state["backend"], + "cuda_tracker": tensor_hash(sorted(state["cuda_tracker"].items()))} + + def audit_state(trainer): import torch - return { + result = { "global_step": trainer.global_step, "state": state_hashes(trainer), "scheduler": cpu_values(trainer.lr_scheduler.state_dict()), "torch_rng_sha256": hashlib.sha256(torch.random.get_rng_state().numpy().tobytes()).hexdigest(), "cuda_rng_sha256": hashlib.sha256(torch.cuda.get_rng_state().cpu().numpy().tobytes()).hexdigest(), } + snapshot = getattr(trainer, "_checkpoint_backend_rng", None) + if snapshot is not None: + result["backend_rng"] = backend_rng_identity(snapshot()) + return result def add_statistics(total, reference, actual): diff --git a/aspec/commands/_recipe_smoke.py b/aspec/commands/_recipe_smoke.py index 57890da..ca07910 100644 --- a/aspec/commands/_recipe_smoke.py +++ b/aspec/commands/_recipe_smoke.py @@ -111,6 +111,11 @@ def init(self, args, role, mooncake_config=None, with_ref=False): } if any(initial[key] != value for key, value in hashes.items()): raise AssertionError("Restored RNG differs from saved checkpoint") + if "backend_rng" in initial: + from aspec.commands._checkpoint_audit import backend_rng_identity + + if backend_rng_identity(rng["backend_rng"]) != initial["backend_rng"]: + raise AssertionError("Restored model-parallel RNG differs from saved checkpoint") (self._audit_root / f"recipe-resume-diff{self._audit_suffix}.json").write_text( json.dumps({"state": compared, "rng_exact": True}, sort_keys=True) ) diff --git a/aspec/commands/doctor.py b/aspec/commands/doctor.py index 410eeaa..a2b2032 100644 --- a/aspec/commands/doctor.py +++ b/aspec/commands/doctor.py @@ -53,8 +53,10 @@ def check_dependency_metadata(distributions=None): return result -def expected_versions(target=False): - source = asset_path("requirements/sglang-v0.5.21.txt" if target else "requirements/draft.txt") +def expected_versions(target=False, *, backend=None): + manifest = ("megatron-v0.20.0.txt" if backend == "megatron" else + "sglang-v0.5.21.txt" if target else "draft.txt") + source = asset_path("requirements/" + manifest) requirements = [Requirement(line) for line in source.read_text().splitlines() if line.strip() and not line.startswith("#")] return {requirement.name: next((spec.version for spec in requirement.specifier @@ -62,7 +64,68 @@ def expected_versions(target=False): for requirement in requirements} -def inspect_environment(*, gpu=False, target=False): +def _inspect_megatron_environment(*, gpu, target): + result = {"python": sys.executable, "prefix": sys.prefix, "platform": platform.platform(), + "gpu_requested": gpu, "target_requested": target, "backend": "megatron", + "role": "target" if target else "draft", "patch": "not_required", + "checks": [], "errors": [], "dependencies": check_dependency_metadata()} + result["errors"].extend(result["dependencies"]["conflicts"]) + if error := venv_error(): + result["errors"].append(error) + if sys.version_info[:2] < (3, 12) or sys.platform != "linux": + result["errors"].append("Megatron requires Linux with Python >=3.12") + for name, expected in expected_versions(backend="megatron").items(): + try: + actual = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + actual = None + ok = actual is not None and (expected is None or actual.split("+", 1)[0] == expected) + result["checks"].append({"package": name, "expected": expected, "installed": actual, "ok": ok}) + if not ok: + result["errors"].append(f"{name}: expected {expected}, found {actual}") + for name in ("ling-aspec", "ling-aspec-target", "sglang"): + try: + importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + pass + else: + result["errors"].append(f"{name} belongs in a separate backend environment") + if gpu: + try: + import torch + import transformer_engine.pytorch as te + from mooncake.engine import TransferEngine + + from aspec.models.megatron import require_megatron + + require_megatron() + layer = te.Linear(16, 16, params_dtype=torch.bfloat16, device="cuda") + output = layer(torch.ones(2, 16, dtype=torch.bfloat16, device="cuda")) + if not torch.isfinite(output).all().item(): + raise RuntimeError("Transformer Engine BF16 linear produced non-finite values") + result["cuda"] = {"torch": torch.__version__, "runtime": torch.version.cuda, + "device_capability": list(torch.cuda.get_device_capability()), + "transformer_engine": te.__file__, + "mooncake_native_import": TransferEngine is not None} + except Exception as error: + result["errors"].append(f"Megatron GPU/native-library check failed: {error}") + result["ok"] = not result["errors"] + return result + + +def inspect_environment(*, gpu=False, target=False, backend=None): + if backend is None: + marker = Path(sys.prefix) / ".aspec-backend" + if marker.is_file() and marker.read_text().strip() == "megatron": + backend = "megatron" + else: + try: + importlib.metadata.version("ling-aspec-megatron") + backend = "megatron" + except importlib.metadata.PackageNotFoundError: + pass + if backend == "megatron": + return _inspect_megatron_environment(gpu=gpu, target=target) result = {"python": sys.executable, "prefix": sys.prefix, "platform": platform.platform(), "gpu_requested": gpu, "target_requested": target, "checks": [], "errors": []} result["dependencies"] = check_dependency_metadata() @@ -137,9 +200,10 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--gpu", action="store_true", help="Also allocate a small CUDA tensor and import Mooncake") parser.add_argument("--target", action="store_true", help="Also require SGLang and its training patch") + parser.add_argument("--backend", choices=("sglang", "megatron"), default=None) parser.add_argument("--json", action="store_true") args = parser.parse_args() - result = inspect_environment(gpu=args.gpu, target=args.target) + result = inspect_environment(gpu=args.gpu, target=args.target, backend=args.backend) if args.json: print(json.dumps(result, indent=2)) else: diff --git a/aspec/config/inference_config.py b/aspec/config/inference_config.py index 1f8a7b0..324bc68 100644 --- a/aspec/config/inference_config.py +++ b/aspec/config/inference_config.py @@ -34,6 +34,15 @@ from aspec.config.mooncake_config import MooncakeConfig +@dataclass +class MegatronConfig: + """Node-local, frozen Qwen3 prefill through upstream Megatron Core.""" + + tp_size: int = 1 + init_timeout: int = 600 + implementation: str = "transformer_engine" + + @dataclass class SGLangConfig: """Essential SGLang engine configuration. @@ -195,6 +204,7 @@ class InferenceConfig: # performance default; the Plan must explicitly admit or omit it. target_prefill_memory_optimized: bool = False offline: OfflineTrainingConfig = field(default_factory=OfflineTrainingConfig) + megatron: MegatronConfig = field(default_factory=MegatronConfig) sglang: SGLangConfig = field(default_factory=SGLangConfig) vllm: VllmConfig = field(default_factory=VllmConfig) trtllm: TrtllmConfig = field(default_factory=TrtllmConfig) diff --git a/aspec/config/megatron.py b/aspec/config/megatron.py new file mode 100644 index 0000000..cc5564e --- /dev/null +++ b/aspec/config/megatron.py @@ -0,0 +1,82 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Dependency-light capability boundary for the upstream Megatron integration.""" + +MEGATRON_VERSION = "0.20.0" +MEGATRON_REVISION = "685fb8bd970185e9ac3283fe32a5d6667bf4b268" + + +def validate_megatron_execution(args, draft): + """Reject combinations outside the runnable dense DSpark CP1/PP1 lane.""" + training = getattr(args, "train_backend", "fsdp") + target = getattr(args, "target_model_backend", "sglang") + tp = getattr(args, "draft_tensor_parallel_size", 1) + if type(tp) is not int or tp < 1: + raise ValueError("training.draft_tensor_parallel_size must be a positive integer") + if training != "megatron" and tp != 1: + raise ValueError("Draft TP requires training.train_backend=megatron") + if training != "megatron" and target != "megatron": + return + if getattr(args, "cluster_mode", "single_node") != "single_node": + raise ValueError("Megatron currently requires single-node execution") + if getattr(args, "aspec_shared_source", False) or getattr(args, "aspec_shared_reader", None): + raise ValueError("Megatron shared Target execution is not supported yet") + if draft["family"] != "dspark": + raise ValueError("Megatron currently supports the DSpark objective") + if draft.get("mtp_conditioning"): + raise ValueError("Megatron currently requires an unconditioned DSpark Draft") + if getattr(args, "draft_context_parallel_size", 1) != 1: + raise ValueError("Megatron currently requires Draft CP1") + if target == "megatron": + if getattr(args, "sglang_extra_args", None) or getattr(args, "sglang_allow_context_length_override", False): + raise ValueError("Megatron Target does not apply SGLang backend overrides") + if getattr(args, "megatron_implementation", "transformer_engine") not in {"local", "transformer_engine"}: + raise ValueError("Unknown Megatron model implementation") + if not getattr(args, "target_prefill_memory_optimized", False): + raise ValueError("Megatron Target requires target_prefill_memory_optimized=true") + if getattr(args, "sglang_nnodes", 1) != 1 or getattr(args, "sglang_pp_size", 1) != 1: + raise ValueError("Megatron Target currently requires node-local TP and PP1") + if getattr(args, "train_with_decode", False): + raise ValueError("Megatron Target supports prefill-only execution") + target_tp = getattr(args, "megatron_tp_size", 1) + if type(target_tp) is not int or target_tp < 1: + raise ValueError("inference.megatron.tp_size must be a positive integer") + if target_tp not in {1, 2}: + raise ValueError("Megatron Target currently validates TP1/TP2") + if target_tp != getattr(args, "inference_num_gpus_per_engine", 1): + raise ValueError("inference.megatron.tp_size must equal inference_num_gpus_per_engine") + slots = (getattr(args, "mooncake_source_gpu_ring_slots", None) + or getattr(args, "mooncake_gpu_ring_slots", 2)) + if getattr(args, "inference_batch_size", 1) > slots: + raise ValueError("Megatron Target source ring must retain one complete inference batch") + if training == "megatron": + if getattr(args, "distributed_backend", "nccl") != "nccl": + raise ValueError("Megatron Draft requires the NCCL distributed backend") + if getattr(args, "micro_batch_size", 1) != 1: + raise ValueError("Megatron Draft currently requires micro_batch_size=1") + prefetch = max(int(getattr(args, "prefetch_depth", 0)), 0) + destination_slots = (getattr(args, "mooncake_destination_gpu_ring_slots", None) + or getattr(args, "mooncake_gpu_ring_slots", 2)) + if prefetch and destination_slots < prefetch + 2: + raise ValueError("Megatron source prefetch requires destination_gpu_ring_slots >= prefetch_depth + 2") + if tp not in {1, 2}: + raise ValueError("Megatron Draft currently validates TP1/TP2") + if getattr(args, "inference_engine_type", "sgl") == "offline": + raise ValueError("Offline replay with Megatron Draft is not validated") + if getattr(args, "fsdp_strategy", "REPLICATE") != "REPLICATE": + raise ValueError("Megatron Draft currently uses replicated DDP rather than FSDP sharding") + if getattr(args, "fsdp_cpu_offload", False): + raise ValueError("Megatron Draft CPU offload is not supported") + if getattr(args, "world_size", 1) % tp: + raise ValueError("Draft world size must be divisible by draft_tensor_parallel_size") + if getattr(args, "dflash_packing", False): + raise ValueError("Megatron Draft packed execution is not supported yet") + if getattr(args, "compile_model", False): + raise ValueError("Megatron Draft currently compiles individual optimized kernels only") + if draft.get("structure", {}).get("ffn_type", "dense") != "dense": + raise ValueError("Megatron Draft currently requires a dense FFN") + attention = draft.get("attention", {}) + if (attention.get("head_layout", {}).get("type", "gqa") not in {"gqa", "mha"} + or any(window is not None for window in attention.get("context_windows", [])) + or any(attention.get(kind) is not None for kind in ("csa", "qsa", "dsa"))): + raise ValueError("Megatron Draft currently requires dense full GQA/MHA attention") diff --git a/aspec/config/train_config.py b/aspec/config/train_config.py index efb4107..62c45a7 100644 --- a/aspec/config/train_config.py +++ b/aspec/config/train_config.py @@ -140,6 +140,7 @@ class TrainingConfig: # Attention backend and context parallelism are independent execution # dimensions. CP>1 currently requires the FA4 backend. draft_context_parallel_size: int = 1 + draft_tensor_parallel_size: int = 1 colocate: bool = False continual_training: bool = False # A weights-only phase transition normally starts a fresh data stream. Set @@ -364,10 +365,11 @@ def _validate_current_scope(config: DictConfig) -> None: """Reject backend selections outside ASpec's current executable path.""" engine_type = config.inference.inference_engine_type target_backend = config.model.target_model_backend - if engine_type not in {"sgl", "offline"} or target_backend != "sglang": + supported = {"sgl": "sglang", "offline": "sglang", "megatron": "megatron"} + if supported.get(engine_type) != target_backend: raise NotImplementedError( - "ASpec currently supports inference.inference_engine_type=sgl or offline with " - "model.target_model_backend=sglang; " + "ASpec currently supports sgl/offline with model.target_model_backend=sglang, " + "or megatron with model.target_model_backend=megatron; " f"got engine_type={engine_type!r}, target_backend={target_backend!r}" ) @@ -849,19 +851,20 @@ def _validate_execution_capabilities(config: DictConfig, *, source_context_paral if not config.inference.target_prefill_memory_optimized: return - if config.inference.inference_engine_type != "sgl": + if config.inference.inference_engine_type not in {"sgl", "megatron"}: raise NotImplementedError( - "target_prefill_memory_optimized currently supports SGLang only" + "target_prefill_memory_optimized requires a live Target backend" ) if config.training.train_with_decode: raise NotImplementedError( "target_prefill_memory_optimized requires prefill-only Target execution" ) - if config.inference.inference_num_gpus_per_engine != 1: + if config.inference.inference_engine_type == "sgl" and config.inference.inference_num_gpus_per_engine != 1: raise NotImplementedError( "target_prefill_memory_optimized currently requires one GPU per Target engine" ) - if config.inference.sglang.tp_size != 1 or config.inference.sglang.pp_size != 1: + if (config.inference.inference_engine_type == "sgl" + and (config.inference.sglang.tp_size != 1 or config.inference.sglang.pp_size != 1)): raise NotImplementedError( "target_prefill_memory_optimized currently requires SGLang TP1/PP1" ) @@ -930,6 +933,10 @@ def _validate_execution_capabilities(config: DictConfig, *, source_context_paral logical_dp_size = world_size // cp_size if is_direct_cp_backend( config.training.attention_backend ) else world_size + tp_size = config.training.draft_tensor_parallel_size if config.training.train_backend == "megatron" else 1 + if type(tp_size) is not int or tp_size < 1: + raise ValueError("training.draft_tensor_parallel_size must be a positive integer") + logical_dp_size //= tp_size dispatch_batch_size = config.training.micro_batch_size * logical_dp_size inference_gpus = config.inference.inference_num_gpus if not isinstance(inference_gpus, int) or inference_gpus < 1: @@ -937,15 +944,16 @@ def _validate_execution_capabilities(config: DictConfig, *, source_context_paral "target_prefill_memory_optimized requires a positive inference_num_gpus" ) if node_local_features: - engine_count = config.inference.inference_num_gpus_per_node + engine_count = (config.inference.inference_num_gpus_per_node + // config.inference.inference_num_gpus_per_engine) dispatch_batch_size = ( config.training.micro_batch_size * config.training.training_num_gpus_per_node - // cp_size + // cp_size // tp_size ) capacity_scope = "per node" else: - engine_count = inference_gpus + engine_count = inference_gpus // config.inference.inference_num_gpus_per_engine capacity_scope = "across the run" resident_sample_capacity = direct_output_sample_capacity( engine_count=engine_count, @@ -1045,6 +1053,7 @@ def load_config( "mooncake": "mooncake_", "offline": "offline_", "sglang": "sglang_", + "megatron": "megatron_", "vllm": "vllm_", "trtllm": "trtllm_", "tokenspeed": "tokenspeed_", diff --git a/aspec/controller/inference_manager.py b/aspec/controller/inference_manager.py index 9018e41..9efe859 100644 --- a/aspec/controller/inference_manager.py +++ b/aspec/controller/inference_manager.py @@ -1216,7 +1216,7 @@ async def leased_engine(): multimodal_inputs=kwargs["multimodal_inputs"], ) if ( - getattr(self.args, "inference_engine_type", "sgl") == "sgl" + getattr(self.args, "inference_engine_type", "sgl") in {"sgl", "megatron"} and not self._train_with_decode ): generate_kwargs["request_ids"] = request_ids diff --git a/aspec/controller/loop.py b/aspec/controller/loop.py index a038e21..8408a59 100644 --- a/aspec/controller/loop.py +++ b/aspec/controller/loop.py @@ -436,7 +436,7 @@ def _safe_training_cleanup( feature_contract_error = None read_owner_fence = closed_reader_fence if train_group is None else None source_teardown_safe = True - external_target = (getattr(args, "inference_engine_type", None) == "sgl" + external_target = (getattr(args, "inference_engine_type", None) in {"sgl", "megatron"} and bool(getattr(args, "target_service", None))) def record(component: str, state: str, error: Exception | None = None) -> None: diff --git a/aspec/evidence.py b/aspec/evidence.py index ccfc32e..0d829d7 100644 --- a/aspec/evidence.py +++ b/aspec/evidence.py @@ -34,6 +34,7 @@ from aspec.planning import ExecutionPlan from aspec.runtime_control import is_termination_exception from aspec.runtime_identity import ( + MEGATRON_RUNTIME_RELATIVE_PATHS, SGLANG_RUNTIME_RELATIVE_PATHS, module_runtime_identity, ) @@ -43,6 +44,9 @@ "ling-aspec", "ling-aspec-runtime", "ling-aspec-target", + "ling-aspec-megatron", + "megatron-core", + "transformer-engine", "torch", "ray", "sglang", @@ -208,7 +212,8 @@ def _module_source_identity(module: str) -> dict[str, Any]: runtime_identity = module_runtime_identity( module, relative_paths=( - SGLANG_RUNTIME_RELATIVE_PATHS if module == "sglang" else None + SGLANG_RUNTIME_RELATIVE_PATHS if module == "sglang" else + MEGATRON_RUNTIME_RELATIVE_PATHS if module == "megatron.core" else None ), ) identity: dict[str, Any] = { @@ -306,7 +311,9 @@ def __init__( "aspec": _source_identity(), "packages": _package_versions(), "package_scope": "draft_driver", - "backends": {"sglang": _module_source_identity("sglang")}, + "backends": ({"megatron": _module_source_identity("megatron.core")} + if plan.policy.backends["target_features"] == "megatron" else + {"sglang": _module_source_identity("sglang")}), "host": { "hostname": platform.node(), "system": platform.system(), @@ -433,13 +440,16 @@ def record_actor_initialization( target_runtimes = [dict(status["runtime"]) for status in (inference_engine_statuses or ()) if isinstance(status.get("runtime"), Mapping)] if target_runtimes: + target_backend = self._plan.policy.backends["target_features"] backend_identities = [item.get("backend_identity") for item in target_runtimes] if (len(target_runtimes) != inference_engine_count + or any(item.get("backend", "sglang") != target_backend for item in target_runtimes) or any(not isinstance(item, Mapping) or not item.get("effective_tree_sha256") or item.get("missing_files") for item in backend_identities) or len({item["effective_tree_sha256"] for item in backend_identities}) != 1): - raise RuntimeError("Target workers reported incomplete or different SGLang identities") - self._data["source"]["backends"]["sglang"] = dict(backend_identities[0]) + name = "SGLang" if target_backend == "sglang" else "Megatron" + raise RuntimeError(f"Target workers reported incomplete or different {name} identities") + self._data["source"]["backends"][target_backend] = dict(backend_identities[0]) self._data["source"]["target_runtimes"] = target_runtimes prior_runtime = self._data["observations"]["runtime"] @@ -500,12 +510,12 @@ def record_actor_initialization( capacity = status.get("token_capacity") if not isinstance(capacity, Mapping): raise RuntimeError( - f"SGLang engine {status.get('rank')} did not report token capacity" + f"Target engine {status.get('rank')} did not report token capacity" ) observed_context = capacity.get("context") if observed_context != planned_context: raise RuntimeError( - "Observed SGLang context contract does not match the Plan: " + "Observed Target context contract does not match the Plan: " f"rank={status.get('rank')}, planned={planned_context}, " f"observed={observed_context}" ) @@ -513,7 +523,7 @@ def record_actor_initialization( "backend_context_length" ]: raise RuntimeError( - "Observed SGLang context length does not match the Plan: " + "Observed Target context length does not match the Plan: " f"rank={status.get('rank')}, " f"planned={backend_tokens['backend_context_length']}, " f"observed={capacity.get('context_length')}" @@ -570,7 +580,7 @@ def record_actor_initialization( { "predicate": ("all_planned_offline_replay_initializers_completed" if self._plan.policy.backends["target_features"] == "offline_replay" - else "all_planned_sglang_engine_initializers_completed"), + else f"all_planned_{self._plan.policy.backends['target_features']}_engine_initializers_completed"), "observed_engine_count": inference_engine_count, }, ) @@ -599,6 +609,18 @@ def _validate_actor_placement( training = [dict(item) for item in (training_actor_statuses or ())] inference = [dict(item) for item in (inference_engine_statuses or ())] + if self._plan.policy.backends["training"] == "megatron": + expected_tp = self._plan.policy.parallelism["training"]["tensor_parallel"] + expected_dp = self._plan.policy.parallelism["training"]["data_parallel"] + if (not training or any(item.get("backend") != "megatron" + or item.get("tensor_parallel_size") != expected_tp + or item.get("data_parallel_size") != expected_dp + or not item.get("initialized") for item in training)): + raise RuntimeError("Megatron trainer/backend TP was not observed as planned") + identities = [item.get("runtime", {}).get("backend_identity", {}) for item in training] + if (any(not item.get("effective_tree_sha256") or item.get("missing_files") for item in identities) + or len({item.get("effective_tree_sha256") for item in identities}) != 1): + raise RuntimeError("Megatron trainers reported incomplete or different Core identities") if training and len(training) != self._plan.policy.resources["training"]["world_size"]: raise RuntimeError("trainer placement status count does not match the Plan") if inference and len(inference) != self._plan.policy.resources["inference"]["engines"]: @@ -833,7 +855,7 @@ def record_training(self, summary: Mapping[str, Any]) -> None: transfer_observation = { "predicate": ("optimizer_step_consumed_validated_offline_features_via_mooncake" if self._plan.policy.backends["target_features"] == "offline_replay" - else "optimizer_step_consumed_validated_sglang_features_via_mooncake"), + else f"optimizer_step_consumed_validated_{self._plan.policy.backends['target_features']}_features_via_mooncake"), "optimizer_steps_observed": training["optimizer_steps_observed"], "feature_contract_id": planned_contract_id, "producer_count": feature_contract["producer_count"], diff --git a/aspec/feature_contract.py b/aspec/feature_contract.py index fdb6d02..7718ace 100644 --- a/aspec/feature_contract.py +++ b/aspec/feature_contract.py @@ -67,6 +67,7 @@ def build_target_feature_contract( target_num_hidden_layers: int, aux_hidden_state_layers: list[int], store_last_hidden_states: bool, + producer_backend: str = "sglang", mtp: Mapping[str, Any] | None = None, ) -> dict[str, Any]: """Build the payload contract without starting a backend. @@ -76,6 +77,8 @@ def build_target_feature_contract( differences still produce distinct identities. """ + if producer_backend not in {"sglang", "megatron"}: + raise FeatureContractError(f"Unsupported Target producer backend: {producer_backend!r}") if not isinstance(target_hidden_size, int) or target_hidden_size <= 0: raise FeatureContractError( f"draft target_hidden_size must be a positive integer, got {target_hidden_size!r}" @@ -164,7 +167,7 @@ def build_target_feature_contract( contract: dict[str, Any] = { "schema_version": FEATURE_CONTRACT_SCHEMA_VERSION, - "producer": {"backend": "sglang", "mode": "prefill_only"}, + "producer": {"backend": producer_backend, "mode": "prefill_only"}, "consumer": { "target_distribution": target_distribution, }, @@ -193,7 +196,10 @@ def validate_feature_contract(contract: Mapping[str, Any]) -> None: raise FeatureContractError( f"Target feature contract identity mismatch: {observed_id!r} != {expected_id!r}" ) - if contract.get("producer") != {"backend": "sglang", "mode": "prefill_only"}: + if contract.get("producer") not in ( + {"backend": "sglang", "mode": "prefill_only"}, + {"backend": "megatron", "mode": "prefill_only"}, + ): raise FeatureContractError( f"Unsupported target feature producer contract: {contract.get('producer')!r}" ) diff --git a/aspec/inference/context.py b/aspec/inference/context.py index b6c034d..59bf8d4 100644 --- a/aspec/inference/context.py +++ b/aspec/inference/context.py @@ -269,6 +269,13 @@ def configure_sglang_context_override(contract: Mapping[str, Any]) -> bool: return enabled +def compile_megatron_context_contract(**kwargs): + """Frozen prefill consumes exactly the requested sequence, without decode reserve.""" + contract = compile_offline_context_contract(**kwargs) + contract["backend_context_length"] = contract["effective_training_max_sequence_length"] + return contract + + def compile_offline_context_contract( *, max_sequence_length, context_parallel_size, model_config, native_context_length, allow_context_length_override, context_shard_alignment=1, diff --git a/aspec/inference/engine/megatron_engine.py b/aspec/inference/engine/megatron_engine.py new file mode 100644 index 0000000..4049e8e --- /dev/null +++ b/aspec/inference/engine/megatron_engine.py @@ -0,0 +1,310 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Frozen upstream Megatron Target workers behind ASpec's Target service. + +Each engine owns a role-local TP world. Only the publishing rank owns source +storage; TP activations are replicated because sequence parallelism is disabled. +Control pipes carry tokens and descriptors, never feature tensors. +""" + +from __future__ import annotations + +import asyncio +import dataclasses +import os +import sys +import threading +import traceback +import uuid +from datetime import timedelta + +import torch + +from aspec.feature_contract import FeatureContractRuntime +from aspec.inference.target_prefill import TargetPrefillDirectSink, TargetPrefillRequest +from aspec.ray.ray_actor import RayActor +from aspec.utils.misc import get_current_node_ip + + +def _target_worker(connection, local_rank, gpu_ids, args, mooncake_config, port, engine_rank): + import torch.distributed as dist + + from aspec.models.megatron import initialize_parallelism, load_qwen3_target + + store = None + try: + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(str(gpu) for gpu in gpu_ids) + torch.cuda.set_device(local_rank) + dist.init_process_group( + "nccl", init_method=f"tcp://127.0.0.1:{port}", + rank=local_rank, world_size=len(gpu_ids), + timeout=timedelta(seconds=getattr(args, "megatron_init_timeout", 600)), + ) + groups = initialize_parallelism(len(gpu_ids)) + model, config = load_qwen3_target( + args.target_model_path, groups, + implementation=getattr(args, "megatron_implementation", "transformer_engine"), + ) + captures = list(args.aux_hidden_states_layers) + runtime = FeatureContractRuntime( + args.aspec_feature_contract, role="producer", participant=f"megatron_engine_{engine_rank}", + ) + runtime.validate_producer_binding( + hidden_size=config.hidden_size, num_hidden_layers=config.num_hidden_layers, + aux_hidden_state_layers=captures, store_last_hidden_states=args.store_last_hidden_states, + ) + active = [None] + if local_rank == 0: + from aspec.transfer.mooncake.eagle_store import EagleMooncakeStore + + source_config = dataclasses.replace(mooncake_config, global_segment_size=0, async_put_pool_size=0) + store = EagleMooncakeStore(source_config) + store.setup(device=local_rank, p2p_role="source") + for layer_id in captures: + def capture(_module, _inputs, output, layer_id=layer_id): + value = output[0] if isinstance(output, tuple) else output + active[0].capture(layer_id, value.squeeze(1), None) + + model.decoder.layers[layer_id].register_forward_hook(capture) + + def finish(*, hidden_states, **_kwargs): + if local_rank == 0: + active[0].finish(hidden_states.squeeze(1)) + return None + + status = { + "pid": os.getpid(), "hidden_size": config.hidden_size, + "python": sys.executable, "prefix": sys.prefix, + "aspec": __import__("aspec").__file__, + "megatron": __import__("megatron.core", fromlist=["__file__"]).__file__, + "torch": torch.__version__, "cuda": torch.version.cuda, + "tensor_parallel_size": len(gpu_ids), "local_rank": local_rank, + "device_capability": list(torch.cuda.get_device_capability()), + "model_implementation": getattr(args, "megatron_implementation", "transformer_engine"), + "target_frozen": not any(p.requires_grad for p in model.parameters()), + } + connection.send({"ok": True, "value": status}) + while True: + method, payload = connection.recv() + if method == "shutdown": + if store is not None: + store.close() + dist.destroy_process_group() + connection.send({"ok": True, "value": None}) + return + if method == "status": + features = runtime.status() + if store is not None: + features["transport"] = store.get_p2p_status() + connection.send({"ok": True, "value": features}) + continue + if method != "generate": + raise ValueError(f"Unknown Megatron Target operation: {method}") + outputs = [] + for sample in payload: + ids = torch.tensor(sample["tokens"], dtype=torch.long, device="cuda") + length = ids.numel() + if not 0 < length <= args.max_seq_length: + raise ValueError("Target input length is outside the planned context") + request = TargetPrefillRequest(sample["request_id"], sample["data_id"], length) + sink = None + try: + if local_rank == 0: + sink = TargetPrefillDirectSink( + store=store, requests=[request], capture_layers=captures, + hidden_size=config.hidden_size, store_last_hidden_states=args.store_last_hidden_states, + source_engine=f"megatron_{engine_rank}", context_parallel_size=1, + ) + active[0] = sink + sink.begin(input_ids=ids, request_ids=[request.request_id], + sequence_lengths=[length], prefix_lengths=[0]) + with torch.inference_mode(): + model(input_ids=ids.unsqueeze(0), + position_ids=torch.arange(length, device="cuda").unsqueeze(0), + attention_mask=None, output_processor=finish) + if local_rank == 0: + key = sink.commit_request(request.request_id) + shapes = {"hidden_states": (length, len(captures) * config.hidden_size), + "input_ids": (length,)} + dtypes = {"hidden_states": torch.bfloat16, "input_ids": torch.int64} + if args.store_last_hidden_states: + shapes["last_hidden_states"] = (length, config.hidden_size) + dtypes["last_hidden_states"] = torch.bfloat16 + metadata = { + "request_id": request.request_id, "engine_rank": engine_rank, + "aspec_feature_contract_id": runtime.contract_id, + "source_node_ip": get_current_node_ip(), + "feature_affinity": getattr(args, "mooncake_feature_affinity", "any"), + "target_feature_output": "direct_to_transport", + "transport_data_plane": "transfer_engine", + "transport_protocol": mooncake_config.protocol, + "feature_layout": "full", + "draft_context_parallel_size": 1, "draft_feature_layout": "full", + "draft_attention_backend": args.attention_backend, + } + from aspec.config.draft_execution import is_direct_cp_backend + + if is_direct_cp_backend(args.attention_backend): + metadata.update(direct_cp_sharded=True, direct_cp_size=1) + runtime.validate_metadata(tensor_shapes=shapes, tensor_dtypes=dtypes, metadata=metadata) + sink.validate_publishable() + sink.publish() + outputs.append({"mooncake_key": key, "tensor_shapes": shapes, + "tensor_dtypes": dtypes, "metadata": metadata}) + except BaseException: + if sink is not None: + sink.abort() + raise + finally: + active[0] = None + connection.send({"ok": True, "value": outputs if local_rank == 0 else None}) + except BaseException as error: + try: + connection.send({"ok": False, "error": repr(error), "traceback": traceback.format_exc()}) + finally: + # Published storage remains owned until the service fences readers + # and observes this process exiting, including native-call failures. + connection.close() + if store is not None: + # Keep all published CUDA registrations alive on failure. The + # Target service terminates this world only after its read fence. + threading.Event().wait() + + +class MegatronEngine(RayActor): + """Ray control facade; model and source storage live in owned child workers.""" + + def __init__(self, args, rank, base_gpu_id=None, num_gpus_per_engine=1, + node_rank=0, engine_group=0, gpu_ids=None): + self.args, self.rank = args, rank + self.base_gpu_id = base_gpu_id + self.gpu_ids = list(gpu_ids or range(base_gpu_id, base_gpu_id + num_gpus_per_engine)) + self.num_gpus_per_engine = num_gpus_per_engine + self._connections, self._processes = [], [] + self._status = [] + self._lock = threading.Lock() + self._mooncake_config = None + self._features = None + self._failed = False + + def _receive(self): + try: + return self._receive_workers() + except BaseException: + self._failed = True + raise + + def _receive_workers(self): + results = [] + for connection, process in zip(self._connections, self._processes, strict=True): + if not connection.poll(getattr(self.args, "megatron_init_timeout", 600)): + raise TimeoutError(f"Megatron Target worker {process.pid} did not complete") + result = connection.recv() + if not result["ok"]: + raise RuntimeError(f"Megatron Target worker failed: {result['error']}\n{result['traceback']}") + results.append(result["value"]) + return results + + def _call(self, method, payload=None): + with self._lock: + if not self.health_check(): + raise RuntimeError("Megatron Target worker world is unavailable") + try: + for connection in self._connections: + connection.send((method, payload)) + return self._receive()[0] + except BaseException: + self._failed = True + raise + + def init(self, mooncake_config=None, pre_allocated_port=None): + if self._processes: + raise RuntimeError("Megatron Target engine cannot be initialized twice") + if mooncake_config is None or mooncake_config.data_plane != "transfer_engine": + raise ValueError("Megatron Target requires the Mooncake transfer_engine data plane") + if pre_allocated_port is None: + raise ValueError("Megatron Target requires a reserved initialization port") + self._mooncake_config = mooncake_config + context = torch.multiprocessing.get_context("spawn") + for rank in range(self.num_gpus_per_engine): + parent, child = context.Pipe() + process = context.Process(target=_target_worker, args=( + child, rank, self.gpu_ids, self.args, mooncake_config, pre_allocated_port, self.rank, + )) + process.start() + child.close() + self._connections.append(parent) + self._processes.append(process) + self._status = self._receive() + + async def generate(self, data_id, input_ids_ref=None, packed_loss_mask_list=None, + formatted_prompts=None, request_ids=None, return_last_hidden_states=False, + return_logits=True, multimodal_inputs=None): + if multimodal_inputs: + raise ValueError("Megatron Target currently supports text-only Qwen3") + if (input_ids_ref is None) == (formatted_prompts is None): + raise ValueError("Provide exactly one of input_ids_ref or formatted_prompts") + if formatted_prompts is not None: + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(self.args.target_model_path, trust_remote_code=False) + tokens = tokenizer(formatted_prompts, add_special_tokens=False)["input_ids"] + else: + tokens = [value.reshape(-1).tolist() for value in input_ids_ref] + ids = ([f"{data_id}_{index}" for index in range(len(tokens))] + if isinstance(data_id, str) else list(data_id)) + if len(ids) != len(tokens): + raise ValueError("Target data identities and token sequences differ in length") + if request_ids is None: + request_ids = [uuid.uuid4().hex for _ in tokens] + elif isinstance(request_ids, str): + request_ids = [f"{request_ids}_{index}" for index in range(len(tokens))] + if len(request_ids) != len(tokens): + raise ValueError("Target request identities and token sequences differ in length") + payload = [dict(tokens=value, data_id=data, request_id=request) + for value, data, request in zip(tokens, ids, request_ids, strict=True)] + return await asyncio.to_thread(self._call, "generate", payload) + + def get_status(self): + return { + "rank": self.rank, "pid": os.getpid(), "node_ip": self.get_node_ip(), + "initialized": bool(self._status) and self.health_check(), + "base_gpu_id": self.base_gpu_id, "engine_gpu_ids": self.gpu_ids, + "num_gpus_per_engine": self.num_gpus_per_engine, + "hidden_size": self._status[0]["hidden_size"] if self._status else None, + "target_prefill_memory_optimized": True, "workers": self._status, + "token_capacity": {"context": self.args.aspec_context_contract, + "context_length": self.args.aspec_context_contract["backend_context_length"], + "profiled_max_total_tokens": None, + "required_max_total_tokens": None, + "max_prefill_tokens": self.args.max_seq_length, + "requested_max_running_requests": 1, + "allocation_policy": "sequential_full_prefill_without_kv_pool"}, + } + + def health_check(self, timeout=5.0): + return (not self._failed and bool(self._processes) + and all(process.is_alive() for process in self._processes)) + + def get_mooncake_config(self): + return self._mooncake_config + + def get_feature_contract_status(self): + if self._processes and self.health_check(): + self._features = self._call("status") + return self._features or {"role": "producer", "state": "pending_observation"} + + async def prepare_feature_contract_status(self): + return await asyncio.to_thread(self.get_feature_contract_status) + + async def shutdown(self): + if self._processes: + await asyncio.to_thread(self._call, "shutdown") + for process in self._processes: + process.join(timeout=30) + if process.is_alive(): + raise RuntimeError("Megatron Target worker did not exit after source drain") + for connection in self._connections: + connection.close() + self._processes.clear() + self._connections.clear() diff --git a/aspec/inference/engine/target_actor.py b/aspec/inference/engine/target_actor.py index 308a294..f5c0b4b 100644 --- a/aspec/inference/engine/target_actor.py +++ b/aspec/inference/engine/target_actor.py @@ -18,9 +18,16 @@ def __init__(self, *args, **kwargs): self.runtime_identity = code_identity() self.gpu_ids = kwargs.pop("gpu_ids") - from aspec.inference.engine.sgl_engine import SglEngine + arguments = kwargs.get("args", args[0] if args else None) + self.backend = getattr(arguments, "target_model_backend", "sglang") + if self.backend == "megatron": + from aspec.inference.engine.megatron_engine import MegatronEngine - self.engine = SglEngine(*args, **kwargs) + self.engine = MegatronEngine(*args, gpu_ids=self.gpu_ids, **kwargs) + else: + from aspec.inference.engine.sgl_engine import SglEngine + + self.engine = SglEngine(*args, **kwargs) def init(self, **kwargs): return self.engine.init(**kwargs) @@ -37,13 +44,19 @@ def find_free_port(self, *args, **kwargs): def get_status(self): import psutil import ray - import sglang import torch import aspec - from aspec.runtime_identity import SGLANG_RUNTIME_RELATIVE_PATHS, module_runtime_identity - - backend = module_runtime_identity("sglang", relative_paths=SGLANG_RUNTIME_RELATIVE_PATHS) + from aspec.runtime_identity import ( + MEGATRON_RUNTIME_RELATIVE_PATHS, + SGLANG_RUNTIME_RELATIVE_PATHS, + module_runtime_identity, + ) + + if self.backend == "megatron": + backend = module_runtime_identity("megatron.core", relative_paths=MEGATRON_RUNTIME_RELATIVE_PATHS) + else: + backend = module_runtime_identity("sglang", relative_paths=SGLANG_RUNTIME_RELATIVE_PATHS) backend_identity = { "path": backend["path"], "git_revision": None, "git_dirty": None, "effective_tree_sha256": backend["effective_tree_sha256"], @@ -52,18 +65,23 @@ def get_status(self): result = self.engine.get_status() result["engine_gpu_ids"] = self.gpu_ids result["created"] = psutil.Process().create_time() + from aspec.inference.target_runtime import dependency_versions + result["runtime"] = { **self.runtime_identity, "python": sys.executable, "prefix": sys.prefix, "aspec": str(Path(aspec.__file__).resolve()), "ray": str(Path(ray.__file__).resolve()), - "sglang": str(Path(sglang.__file__).resolve()), "backend_identity": backend_identity, - "sglang_version": importlib.metadata.version("sglang"), + "backend": self.backend, + "dependencies": dependency_versions("sglang", "megatron-core", "transformer-engine"), "torch": torch.__version__, "cuda": torch.version.cuda, - "cutlass": importlib.metadata.version("nvidia-cutlass-dsl"), - "quack": importlib.metadata.version("quack-kernels"), + "cutlass": dependency_versions("nvidia-cutlass-dsl")["nvidia-cutlass-dsl"], + "quack": dependency_versions("quack-kernels")["quack-kernels"], } + if self.backend == "sglang": + result["runtime"]["sglang"] = backend["path"] + result["runtime"]["sglang_version"] = importlib.metadata.version("sglang") return result def get_mooncake_config(self): diff --git a/aspec/inference/factory.py b/aspec/inference/factory.py index 45fdccc..9305f81 100644 --- a/aspec/inference/factory.py +++ b/aspec/inference/factory.py @@ -37,10 +37,9 @@ def _require_supported_engine(engine_type: str) -> None: - if engine_type not in {"sgl", "offline"}: + if engine_type not in {"sgl", "offline", "megatron"}: raise NotImplementedError( - "ASpec currently supports the SGLang or offline replay path " - f"(inference_engine_type='sgl' or 'offline'); got {engine_type!r}" + f"Supported inference engines are sgl, megatron and offline; got {engine_type!r}" ) @@ -59,7 +58,7 @@ def create_inference_engines(args, inference_pg, mooncake_config, engine_group: _wait_for_init(init_refs, "OfflineReplay", timeout=300) return engines - if engine_type not in ("hf", "sgl", "vllm", "trtllm", "tokenspeed"): + if engine_type not in ("hf", "sgl", "vllm", "trtllm", "tokenspeed", "megatron"): raise ValueError(f"Unknown inference_engine_type: {engine_type}") logger.info(f"Using {engine_type} engine for inference") @@ -94,14 +93,14 @@ def prepare_inference_engines(args, inference_pg, mooncake_config, engine_group: if engine_type == "offline": return _prepare_offline_replay_engines(args, mooncake_config, engine_group) - if engine_type not in ("hf", "sgl", "vllm", "trtllm", "tokenspeed"): + if engine_type not in ("hf", "sgl", "vllm", "trtllm", "tokenspeed", "megatron"): raise ValueError(f"Unknown inference_engine_type: {engine_type}") logger.info(f"Preparing {engine_type} inference engines...") if engine_type == "hf": engines, init_refs = _prepare_hf_engines(args, inference_pg, mooncake_config, engine_group) - elif engine_type == "sgl": + elif engine_type in {"sgl", "megatron"}: engines, init_refs = _prepare_sgl_engines(args, inference_pg, mooncake_config, engine_group) elif engine_type == "trtllm": engines, init_refs = _prepare_trtllm_engines( @@ -159,7 +158,7 @@ def init_engines(args, pg, engine_type: str, mooncake_config=None, engine_group: _require_supported_engine(engine_type) if engine_type == "hf": return _init_hf_engines(args, pg, mooncake_config, engine_group) - elif engine_type == "sgl": + elif engine_type in {"sgl", "megatron"}: return _init_sgl_engines(args, pg, mooncake_config, engine_group) elif engine_type == "vllm": return _init_vllm_engines(args, pg, mooncake_config, engine_group) diff --git a/aspec/inference/managed.py b/aspec/inference/managed.py index 47a89ca..f2c12e3 100644 --- a/aspec/inference/managed.py +++ b/aspec/inference/managed.py @@ -53,7 +53,8 @@ def managed_target(args, *, draft_gpu_count=None): config = root / "target-service.json" config.write_text(json.dumps({"model": {"target_model_path": args.target_model_path}})) command = [settings["python_executable"], "-m", "aspec._target_worker", "-m", - "aspec.inference.service", "--config", str(config), "--address", address] + "aspec.inference.service", "--config", str(config), "--address", address, + "--backend", getattr(args, "target_model_backend", "sglang")] log_path = root / "target-service.log" with log_path.open("w") as log: process = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, @@ -61,7 +62,10 @@ def managed_target(args, *, draft_gpu_count=None): ready = False prior_devices = os.environ.get("CUDA_VISIBLE_DEVICES") try: - deadline = time.monotonic() + float(getattr(args, "sglang_init_timeout", 300)) + timeout_field = ("megatron_init_timeout" + if getattr(args, "target_model_backend", "sglang") == "megatron" + else "sglang_init_timeout") + deadline = time.monotonic() + float(getattr(args, timeout_field, 300)) while time.monotonic() < deadline: if process.poll() is not None: raise RuntimeError(f"Target exited before readiness; see {log_path}") diff --git a/aspec/inference/service.py b/aspec/inference/service.py index af37079..5b6b696 100644 --- a/aspec/inference/service.py +++ b/aspec/inference/service.py @@ -93,6 +93,8 @@ def create(self, value): if set(arguments) - TARGET_ARGUMENTS: raise ValueError("Unknown Target configuration fields") args = Namespace(**arguments) + if getattr(args, "target_model_backend", "sglang") != self.runtime.get("backend", "sglang"): + raise ValueError("Requested Target backend differs from the service environment") validate_target_tp(getattr(args, "sglang_nnodes", 1), value["node_rank"]) if self.model and args.target_model_path != self.model: raise ValueError("Target model differs from the service configuration") @@ -437,6 +439,7 @@ def main(): logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser(description="Run Target in this environment; Draft connects by address.") parser.add_argument("--config", type=Path) + parser.add_argument("--backend", choices=("sglang", "megatron"), default="sglang") parser.add_argument("--address", help="Listening http://host:port; defaults to the config or localhost:30000") args = parser.parse_args() config = OmegaConf.load(args.config) if args.config else OmegaConf.create({}) @@ -446,7 +449,7 @@ def main(): devices = visible_gpu_ids() if not devices: parser.error("Select allocated Target GPUs with CUDA_VISIBLE_DEVICES") - runtime = inventory() # Check this environment and patch before starting processes. + runtime = inventory(args.backend) service = TargetService(model=model, devices=devices, runtime=runtime) parsed = urlsplit(address) server = http_server(service, parsed.hostname, parsed.port) diff --git a/aspec/inference/service_protocol.py b/aspec/inference/service_protocol.py index 9de6d54..c66f6e0 100644 --- a/aspec/inference/service_protocol.py +++ b/aspec/inference/service_protocol.py @@ -23,7 +23,8 @@ # to this interface. Optimizers, datasets, logging credentials and driver # objects remain in Draft. Unknown Draft settings do not change this protocol. TARGET_ARGUMENTS = frozenset(""" -target_model_path trust_remote_code aux_hidden_states_layers store_last_hidden_states native_context_length +target_model_path target_model_backend trust_remote_code aux_hidden_states_layers store_last_hidden_states native_context_length +megatron_tp_size megatron_init_timeout megatron_implementation draft_algorithm mooncake_hidden_dim sglang_mem_fraction_static sglang_pp_size sglang_nnodes sglang_tp_size sglang_extra_args sglang_allow_context_length_override sglang_enable_multimodal diff --git a/aspec/inference/target_runtime.py b/aspec/inference/target_runtime.py index 3527c93..e1b94f9 100644 --- a/aspec/inference/target_runtime.py +++ b/aspec/inference/target_runtime.py @@ -35,8 +35,18 @@ def dependency_versions(*names): return result -def inventory(): +def inventory(backend="sglang"): """Read Target identity in its own interpreter, without starting GPU services.""" + if backend == "megatron": + from aspec.models.megatron import require_megatron + from aspec.runtime_identity import MEGATRON_RUNTIME_RELATIVE_PATHS, module_runtime_identity + + require_megatron() + return {**code_identity(), "backend": backend, "python": sys.executable, "prefix": sys.prefix, + "megatron": module_runtime_identity("megatron.core", relative_paths=MEGATRON_RUNTIME_RELATIVE_PATHS), + "dependencies": dependency_versions("torch", "ray", "megatron-core", "transformer-engine")} + if backend != "sglang": + raise ValueError(f"Unsupported Target backend: {backend}") from aspec.commands.doctor import inspect_environment from aspec.runtime_identity import SGLANG_RUNTIME_RELATIVE_PATHS, module_runtime_identity diff --git a/aspec/models/megatron.py b/aspec/models/megatron.py new file mode 100644 index 0000000..8e6b619 --- /dev/null +++ b/aspec/models/megatron.py @@ -0,0 +1,257 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Public Megatron Core construction and strict Qwen3 weight mapping. + +Uses Megatron Core 0.20's public model/spec/process-group interfaces. Qwen3 +mapping follows the grouped QKV and gate/up layouts documented by NVIDIA's +Megatron Bridge Qwen3 adapter; no internal Megatron implementation is imported. +""" + +from __future__ import annotations + +import importlib.metadata +import json +from dataclasses import replace +from functools import partial +from pathlib import Path + +import torch +import torch.distributed as dist +from safetensors import safe_open + +from aspec.config.megatron import MEGATRON_VERSION +from aspec.models.target.target_utils import _TargetRMSNorm + + +class Qwen3RMSNorm(_TargetRMSNorm): + """Retain HF's normalize/cast/weight order through a public MCore spec. + + Reuses ASpec's retained TorchSpec Target norm. Fusing the weight multiply + before the BF16 cast changes Qwen3's intermediate residual features. + """ + + returns_residual = False + + def __init__(self, *, config, hidden_size, eps): + super().__init__(hidden_size, eps) + + +def require_megatron(): + from packaging.version import Version + + try: + version = importlib.metadata.version("megatron-core") + except importlib.metadata.PackageNotFoundError as exc: + raise RuntimeError("Install the pinned ASpec Megatron environment") from exc + if Version(version).base_version != MEGATRON_VERSION: + raise RuntimeError(f"Megatron Core {MEGATRON_VERSION} is required; found {version}") + return version + + +def initialize_parallelism(tp_size: int, *, seed: int = 42): + """The caller's default process group contains only one execution role.""" + require_megatron() + from megatron.core import parallel_state + from megatron.core.process_groups_config import ProcessGroupCollection + + if not dist.is_initialized(): + raise RuntimeError("Initialize the role-local process group before Megatron") + if parallel_state.model_parallel_is_initialized(): + raise RuntimeError("A Megatron model-parallel world is already initialized") + parallel_state.initialize_model_parallel( + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=1, + context_parallel_size=1, + create_gloo_process_groups=False, + ) + from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed + + model_parallel_cuda_manual_seed(seed) + return ProcessGroupCollection.use_mpu_process_groups() + + +def transformer_config(config, tp_size: int): + from megatron.core.transformer.transformer_config import TransformerConfig + + hidden = int(config.hidden_size) + heads = int(config.num_attention_heads) + groups = int(config.num_key_value_heads) + intermediate = int(config.intermediate_size) + if any(value % tp_size for value in (heads, groups, intermediate, config.vocab_size)): + raise ValueError("Qwen3 heads, KV groups, FFN and vocabulary must divide Target TP") + if config.model_type != "qwen3" or getattr(config, "num_experts", None): + raise ValueError("Megatron Target currently requires dense Qwen3") + if getattr(config, "quantization_config", None) or getattr(config, "hidden_act", "silu") != "silu": + raise ValueError("Megatron Qwen3 requires unquantized dense SwiGLU weights") + rope = getattr(config, "rope_parameters", None) or {} + scaling = getattr(config, "rope_scaling", None) or rope + if scaling.get("rope_type", scaling.get("type", "default")) != "default": + raise ValueError("Megatron Qwen3 currently requires the original default RoPE policy") + return TransformerConfig( + num_layers=int(config.num_hidden_layers), + hidden_size=hidden, + ffn_hidden_size=intermediate, + num_attention_heads=heads, + num_query_groups=groups, + kv_channels=int(getattr(config, "head_dim", None) or hidden // heads), + normalization="RMSNorm", + layernorm_epsilon=float(config.rms_norm_eps), + qk_layernorm=True, + gated_linear_unit=True, + activation_func=torch.nn.functional.silu, + add_bias_linear=False, + add_qkv_bias=False, + attention_dropout=0.0, + hidden_dropout=0.0, + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=1, + context_parallel_size=1, + params_dtype=torch.bfloat16, + bf16=True, + perform_initialization=False, + use_cpu_initialization=False, + gradient_accumulation_fusion=False, + ) + + +class SafetensorReader: + """Read one source parameter at a time, with an explicit model identity.""" + + def __init__(self, path): + from huggingface_hub import snapshot_download + + self.root = Path(path) + if not self.root.is_dir(): + self.root = Path(snapshot_download(path, allow_patterns=["*.json", "*.safetensors"])) + index = self.root / "model.safetensors.index.json" + if index.is_file(): + self.files = json.loads(index.read_text())["weight_map"] + else: + with safe_open(self.root / "model.safetensors", framework="pt", device="cpu") as source: + self.files = {key: "model.safetensors" for key in source.keys()} + self.read = set() + + def tensor(self, name): + if name not in self.files: + raise ValueError(f"Target checkpoint is missing required parameter {name}") + self.read.add(name) + with safe_open(self.root / self.files[name], framework="pt", device="cpu") as source: + return source.get_tensor(name) + + +def _shard(value, axis, rank, size): + if value.shape[axis] % size: + raise ValueError(f"Tensor dimension {axis} cannot be partitioned over TP{size}") + return value.chunk(size, dim=axis)[rank].contiguous() + + +def qwen3_parameter(reader, name, config, rank, tp_size): + """Map public MCore parameter names, preserving native Qwen3 values.""" + if name == "embedding.word_embeddings.weight": + return _shard(reader.tensor("model.embed_tokens.weight"), 0, rank, tp_size) + if name == "output_layer.weight": + source = "model.embed_tokens.weight" if config.tie_word_embeddings else "lm_head.weight" + return _shard(reader.tensor(source), 0, rank, tp_size) + if name == "decoder.final_layernorm.weight": + return reader.tensor("model.norm.weight") + pieces = name.split(".") + if pieces[:2] != ["decoder", "layers"] or len(pieces) < 5: + raise ValueError(f"Unsupported Megatron Qwen3 parameter: {name}") + prefix = f"model.layers.{int(pieces[2])}." + suffix = ".".join(pieces[3:]) + aliases = { + "input_layernorm.weight": "input_layernorm.weight", + "self_attention.linear_qkv.layer_norm_weight": "input_layernorm.weight", + "pre_mlp_layernorm.weight": "post_attention_layernorm.weight", + "mlp.linear_fc1.layer_norm_weight": "post_attention_layernorm.weight", + "self_attention.q_layernorm.weight": "self_attn.q_norm.weight", + "self_attention.k_layernorm.weight": "self_attn.k_norm.weight", + } + if suffix in aliases: + return reader.tensor(prefix + aliases[suffix]) + if suffix == "self_attention.linear_qkv.weight": + groups = int(config.num_key_value_heads) + heads_per_group = int(config.num_attention_heads) // groups + width = int(getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads) + query = reader.tensor(prefix + "self_attn.q_proj.weight").view(groups, heads_per_group, width, -1) + key = reader.tensor(prefix + "self_attn.k_proj.weight").view(groups, 1, width, -1) + value = reader.tensor(prefix + "self_attn.v_proj.weight").view(groups, 1, width, -1) + return _shard(torch.cat((query, key, value), dim=1), 0, rank, tp_size).flatten(0, 2) + if suffix == "self_attention.linear_proj.weight": + return _shard(reader.tensor(prefix + "self_attn.o_proj.weight"), 1, rank, tp_size) + if suffix == "mlp.linear_fc1.weight": + return torch.cat([ + _shard(reader.tensor(prefix + "mlp." + part + ".weight"), 0, rank, tp_size) + for part in ("gate_proj", "up_proj") + ], dim=0) + if suffix == "mlp.linear_fc2.weight": + return _shard(reader.tensor(prefix + "mlp.down_proj.weight"), 1, rank, tp_size) + raise ValueError(f"Unsupported Megatron Qwen3 parameter: {name}") + + +def load_qwen3_target(path, groups, *, implementation="transformer_engine"): + """Construct an upstream GPTModel and load every parameter strictly.""" + from megatron.core.models.backends import get_backend + from megatron.core.models.gpt.gpt_layer_specs import ( + get_gpt_layer_local_spec, + get_gpt_layer_with_transformer_engine_spec, + ) + from megatron.core.models.gpt.gpt_model import GPTModel + from megatron.core.transformer.mlp import MLP, MLPSubmodules + from megatron.core.transformer.transformer_block import TransformerBlockSubmodules + from transformers import AutoConfig + + reader = SafetensorReader(path) + config = AutoConfig.from_pretrained(reader.root, trust_remote_code=False) + tp_size = dist.get_world_size(groups.tp) + rank = dist.get_rank(groups.tp) + core_config = transformer_config(config, tp_size) + if implementation == "transformer_engine": + spec = get_gpt_layer_with_transformer_engine_spec(qk_layernorm=True) + elif implementation == "local": + spec = get_gpt_layer_local_spec(qk_layernorm=True, normalization="RMSNorm") + else: + raise ValueError(f"Unknown Megatron model implementation {implementation}") + backend = get_backend(implementation) + attention = spec.submodules.self_attention + attention = replace(attention, submodules=replace( + attention.submodules, linear_qkv=backend.column_parallel_linear(), + q_layernorm=Qwen3RMSNorm, k_layernorm=Qwen3RMSNorm, + )) + mlp = partial(MLP.as_mlp_submodule, submodules=MLPSubmodules( + linear_fc1=backend.column_parallel_linear(), linear_fc2=backend.row_parallel_linear(), + )) + spec = replace(spec, submodules=replace( + spec.submodules, input_layernorm=Qwen3RMSNorm, self_attention=attention, + pre_mlp_layernorm=Qwen3RMSNorm, mlp=mlp, + )) + block = TransformerBlockSubmodules( + layer_specs=[spec] * core_config.num_layers, layer_norm=Qwen3RMSNorm, + ) + rope = getattr(config, "rope_parameters", None) or {} + with torch.device("cuda"): + model = GPTModel( + config=core_config, + transformer_layer_spec=block, + vocab_size=config.vocab_size, + max_sequence_length=config.max_position_embeddings, + share_embeddings_and_output_weights=bool(config.tie_word_embeddings), + position_embedding_type="rope", + rotary_percent=1.0, + rotary_base=rope.get("rope_theta", getattr(config, "rope_theta", 1000000.0)), + pg_collection=groups, + ) + model.to(dtype=torch.bfloat16) + with torch.no_grad(): + for name, parameter in model.named_parameters(): + value = qwen3_parameter(reader, name, config, rank, tp_size) + if tuple(value.shape) != tuple(parameter.shape): + raise ValueError(f"Qwen3 mapping shape mismatch for {name}: {value.shape} != {parameter.shape}") + parameter.copy_(value) + unused = set(reader.files) - reader.read + if config.tie_word_embeddings: + unused.discard("lm_head.weight") + if unused: + raise ValueError(f"Unmapped Target checkpoint parameters: {sorted(unused)}") + model.eval().requires_grad_(False) + return model, config diff --git a/aspec/models/megatron_draft.py b/aspec/models/megatron_draft.py new file mode 100644 index 0000000..970cccc --- /dev/null +++ b/aspec/models/megatron_draft.py @@ -0,0 +1,121 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Megatron TP projections around the retained optimized DSpark computation. + +The vocabulary objective and its bounded/tiled caches remain replicated in this +CP1 implementation. Only attention heads and dense FFN channels are partitioned. +This avoids gathering vocabulary logits or changing global probability semantics. +""" + +import torch +from torch.distributed.tensor import DTensor, Shard + +from aspec.models.megatron import require_megatron + + +def draft_transformer_config(config, tp_size): + require_megatron() + from megatron.core.transformer.transformer_config import TransformerConfig + + for name in ("num_attention_heads", "num_key_value_heads", "intermediate_size"): + if getattr(config, name) % tp_size: + raise ValueError(f"Draft {name} must be divisible by TP{tp_size}") + return TransformerConfig( + num_layers=config.num_hidden_layers, hidden_size=config.hidden_size, + ffn_hidden_size=config.intermediate_size, + num_attention_heads=config.num_attention_heads, + num_query_groups=config.num_key_value_heads, + kv_channels=int(getattr(config, "head_dim", None) + or config.hidden_size // config.num_attention_heads), normalization="RMSNorm", + layernorm_epsilon=config.rms_norm_eps, gated_linear_unit=True, + activation_func=torch.nn.functional.silu, add_bias_linear=False, + params_dtype=torch.bfloat16, bf16=True, + tensor_model_parallel_size=tp_size, context_parallel_size=1, + pipeline_model_parallel_size=1, sequence_parallel=False, + perform_initialization=False, use_cpu_initialization=False, + gradient_accumulation_fusion=False, attention_dropout=0.0, hidden_dropout=0.0, + ) + + +def checkpoint_tensor(tensor, parameter, tp_mesh): + """Expose the logical tensor to DCP without changing compute parameters.""" + axis = getattr(parameter, "_aspec_tp_axis", None) + if isinstance(tensor, DTensor) or axis is None or tensor.ndim == 0: + return tensor + return DTensor.from_local(tensor, device_mesh=tp_mesh, placements=(Shard(axis),), + run_check=False) + + +def partition_dspark(draft, core_config, groups, tp_mesh): + from megatron.core.tensor_parallel.layers import ColumnParallelLinear, RowParallelLinear + + class Column(ColumnParallelLinear): + def forward(self, value): + output, bias = super().forward(value) + if bias is not None: + raise RuntimeError("DSpark TP projections must be bias-free") + return output + + class Row(RowParallelLinear): + def forward(self, value): + output, bias = super().forward(value) + if bias is not None: + raise RuntimeError("DSpark TP projections must be bias-free") + return output + + rank, size = groups.tp.rank(), groups.tp.size() + + def replace(owner, name, axis): + original = getattr(owner, name) + if original.bias is not None: + raise ValueError("Megatron DSpark requires bias-free projections") + options = {"gather_output": False} if axis == 0 else {"input_is_parallel": True} + layer_type = Column if axis == 0 else Row + replacement = layer_type( + original.in_features, original.out_features, config=core_config, + init_method=core_config.init_method, bias=False, skip_bias_add=False, + tp_group=groups.tp, **options, + ) + with torch.no_grad(): + replacement.weight.copy_(original.weight.chunk(size, dim=axis)[rank]) + replacement.weight.requires_grad_(original.weight.requires_grad) + replacement.weight._aspec_tp_axis = axis + setattr(owner, name, replacement) + + for layer in draft.layers: + attention = layer.self_attn + for name in ("q_proj", "k_proj", "v_proj"): + replace(attention, name, 0) + replace(attention, "o_proj", 1) + attention.num_heads //= size + attention.num_kv_heads //= size + for name in ("gate_proj", "up_proj"): + replace(layer.mlp, name, 0) + replace(layer.mlp, "down_proj", 1) + draft.cuda() + for layer in draft.layers: + for norm in (layer.self_attn.q_norm, layer.self_attn.k_norm): + norm.weight._aspec_tp_partial_gradient = True + draft._aspec_tp_mesh = tp_mesh + draft._aspec_tp_group = groups.tp + return draft + + +def model_checkpoint_state(model, tp_mesh): + parameters = dict(model.named_parameters()) + return {name: checkpoint_tensor(value, parameters[name], tp_mesh) + if name in parameters else value + for name, value in model.state_dict().items() if isinstance(value, torch.Tensor)} + + +def load_model_checkpoint_state(model, state): + current = {name: value for name, value in model.state_dict().items() + if isinstance(value, torch.Tensor)} + if set(current) != set(state): + raise ValueError("Megatron checkpoint does not cover the complete DSpark state") + with torch.no_grad(): + for name, value in state.items(): + local = value.to_local() if isinstance(value, DTensor) else value + if current[name].shape != local.shape: + raise ValueError(f"Megatron checkpoint shape mismatch: {name}") + current[name].copy_(local) diff --git a/aspec/planning.py b/aspec/planning.py index 29542fc..f15a705 100644 --- a/aspec/planning.py +++ b/aspec/planning.py @@ -47,12 +47,14 @@ resolve_draft_execution_capability, ) from aspec.config.draft_structure import resolve_draft_structure +from aspec.config.megatron import validate_megatron_execution from aspec.config.mooncake_config import MooncakeConfig, resolve_rdma_routes from aspec.config.train_config import config_to_flat_args from aspec.feature_contract import FeatureContractError, build_target_feature_contract from aspec.inference.concurrency import direct_output_sample_capacity from aspec.inference.context import ( ContextContractError, + compile_megatron_context_contract, compile_offline_context_contract, compile_sglang_context_contract, ) @@ -708,7 +710,16 @@ def _resolve_batch_size(args: argparse.Namespace) -> None: ) args.attention_backend = execution.attention_backend args.draft_context_parallel_size = execution.context_parallel_size - if execution.direct_cp_backend: + if getattr(args, "train_backend", "fsdp") == "megatron": + tp_size = getattr(args, "draft_tensor_parallel_size", 1) + if type(tp_size) is not int or tp_size < 1 or world_size % tp_size: + raise PlanCompileError("Draft world size must be divisible by positive draft_tensor_parallel_size") + dp_size = world_size // tp_size + args.dp_size = dp_size + args.sp_size = 1 + args.per_dp_rank_batch_size = args.micro_batch_size + sp_size = 1 + elif execution.direct_cp_backend: cp_size = execution.context_parallel_size if world_size % cp_size != 0: raise PlanCompileError( @@ -803,10 +814,14 @@ def _validate_usp_args(args: argparse.Namespace) -> None: def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None: - if getattr(args, "train_backend", None) != "fsdp": + if getattr(args, "train_backend", None) not in {"fsdp", "megatron"}: raise PlanCompileError( - f"The current training backend is fsdp; got {getattr(args, 'train_backend', None)!r}" + f"Training backend must be fsdp or megatron; got {getattr(args, 'train_backend', None)!r}" ) + try: + validate_megatron_execution(args, draft) + except ValueError as exc: + raise PlanCompileError(str(exc)) from exc cluster_mode = getattr(args, "cluster_mode", "single_node") if cluster_mode not in {"single_node", "static_multi_node"}: raise PlanCompileError(f"Unsupported cluster.mode {cluster_mode!r}") @@ -909,7 +924,10 @@ def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None: raise PlanCompileError("static_multi_node requires cluster.shared_root") expected_tp = getattr(args, "sglang_nnodes", 1) * gpus_per_engine - if not offline and getattr(args, "sglang_tp_size", None) != expected_tp: + target_tp = (getattr(args, "megatron_tp_size", 1) + if getattr(args, "target_model_backend", "sglang") == "megatron" + else getattr(args, "sglang_tp_size", None)) + if not offline and target_tp != expected_tp: raise PlanCompileError( "inference.sglang.tp_size must equal sglang.nnodes * " f"inference_num_gpus_per_engine ({expected_tp})" @@ -1062,15 +1080,16 @@ def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None: getattr(args, "target_prefill_memory_optimized", False) ) if direct_target_output: - if getattr(args, "inference_engine_type", "sgl") != "sgl": + if getattr(args, "inference_engine_type", "sgl") not in {"sgl", "megatron"}: raise PlanCompileError( - "target_prefill_memory_optimized currently requires SGLang" + "target_prefill_memory_optimized requires a live Target backend" ) if getattr(args, "train_with_decode", False): raise PlanCompileError( "target_prefill_memory_optimized requires prefill-only Target execution" ) - if gpus_per_engine != 1 or getattr(args, "sglang_tp_size", 1) != 1: + if (getattr(args, "inference_engine_type", "sgl") == "sgl" + and (gpus_per_engine != 1 or getattr(args, "sglang_tp_size", 1) != 1)): raise PlanCompileError( "target_prefill_memory_optimized currently requires Target TP1" ) @@ -1132,6 +1151,7 @@ def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None: dispatch_batch_size = ( int(getattr(args, "training_num_gpus_per_node", 0)) // int(getattr(args, "draft_context_parallel_size", 1)) + // int(getattr(args, "draft_tensor_parallel_size", 1)) * args.per_dp_rank_batch_size ) capacity_scope = "per node" @@ -1387,6 +1407,7 @@ def _workload( target_num_hidden_layers=draft["target_num_hidden_layers"], aux_hidden_state_layers=list(draft["target_layer_ids"]), store_last_hidden_states=store_last, + producer_backend=_select(config, "model.target_model_backend", "sglang"), **({"mtp": {"native_config": draft["mtp_conditioning"]["native_config"], "embedding_key": _select(config, "model.embedding_key")}} if draft.get("mtp_conditioning") and draft["mtp_conditioning"]["mode"] == "hidden" @@ -1513,6 +1534,22 @@ def _policy( _select(config, "inference.inference_batch_size"), ), ) + megatron_input_memory = None + if runtime.get("train_backend") == "megatron": + # Receive/collate copies on the consumer thread. Leader source prefetch + # borrows registered destination slots, counted separately as rings. + resident_batches = 2 + feature_width = int(_select(config, "mooncake.hidden_dim", 4096)) + feature_bytes_per_token = 2 * feature_width * ( + len(runtime["aux_hidden_states_layers"]) + int(runtime.get("store_last_hidden_states", False))) + # Token IDs plus masks/positions alongside the captured feature ABI. + bytes_per_batch = (feature_bytes_per_token + 40) * int(runtime["max_seq_length"]) * int(runtime["micro_batch_size"]) + megatron_input_memory = { + "storage": "consumer_cuda_payloads_replicated_over_tp", + "resident_batch_bound_per_rank": resident_batches, + "payload_bytes_bound_per_rank": resident_batches * bytes_per_batch, + "excludes": "registered_rings_model_activations_and_projection_caches", + } cluster_nodes: list[dict[str, Any]] = [] locality_domains: list[dict[str, Any]] = [] ready_watermark_by_domain: dict[str, int] = {} @@ -1629,7 +1666,8 @@ def _policy( return ExecutionPolicy( backends={ - "target_features": "offline_replay" if runtime["inference_engine_type"] == "offline" else "sglang", + "target_features": ("offline_replay" if runtime["inference_engine_type"] == "offline" + else _select(config, "model.target_model_backend")), "training": _select(config, "training.train_backend"), "transport": "mooncake", }, @@ -1658,7 +1696,7 @@ def _policy( "world_size": runtime["world_size"], }, "inference": { - "gpu_owner": "target_service" if runtime["inference_engine_type"] == "sgl" else None, + "gpu_owner": "target_service" if runtime["inference_engine_type"] != "offline" else None, "nodes": ( len(_select(config, "inference.inference_node_ips") or ()) if cluster_mode == "static_multi_node" @@ -1679,15 +1717,21 @@ def _policy( parallelism={ "training": { "data_parallel": runtime["dp_size"], + "tensor_parallel": _select(config, "training.draft_tensor_parallel_size", 1), "sequence_parallel": runtime["sp_size"], "context_parallel": _select( config, "training.draft_context_parallel_size", 1 ), "attention_backend": _select(config, "training.attention_backend"), - "fsdp_strategy": _select(config, "training.fsdp_strategy"), + "fsdp_strategy": (None if runtime["train_backend"] == "megatron" + else _select(config, "training.fsdp_strategy")), + **({"gradient_reduce_dtype": "float32", "vocabulary": "replicated"} + if runtime["train_backend"] == "megatron" else {}), }, "inference": { - "tensor_parallel": _select(config, "inference.sglang.tp_size"), + "tensor_parallel": (_select(config, "inference.megatron.tp_size") + if runtime["inference_engine_type"] == "megatron" + else _select(config, "inference.sglang.tp_size")), "pipeline_parallel": _select(config, "inference.sglang.pp_size"), "source_context_parallel_size": runtime["source_context_parallel_size"], }, @@ -1731,15 +1775,18 @@ def _policy( }, capacity={ "device_memory": { + **({"megatron_input_staging": megatron_input_memory} if megatron_input_memory else {}), **({"mtp_conditioning": conditioning_memory} if conditioning_memory else {}), **({"target_mtp_features": target_mtp_memory} if target_mtp_memory else {}), - "sglang_static_fraction": (None if runtime["inference_engine_type"] == "offline" + "sglang_static_fraction": (None if runtime["inference_engine_type"] != "sgl" else _select(config, "inference.sglang.mem_fraction_static")) }, "backend_tokens": { **copy.deepcopy(dict(context_contract)), "profiled_max_total_tokens": None, - "state": "not_applicable_offline" if runtime["inference_engine_type"] == "offline" else "pending_backend_probe", + "state": ("not_applicable_offline" if runtime["inference_engine_type"] == "offline" + else "no_persistent_kv_pool" if runtime["inference_engine_type"] == "megatron" + else "pending_backend_probe"), }, "transfer_storage": { "data_plane": data_plane, @@ -1871,9 +1918,18 @@ def compile_execution_plan( source["resolved_config_sha256"] = _resolved_config_sha256(config) draft = _load_draft_descriptor(config) target_family = _resolve_target_family(target_path) + if (_select(config, "training.train_backend") == "megatron" + or _select(config, "model.target_model_backend") == "megatron"): + target_raw = _read_local_target_config(target_path) + if target_family != "qwen3" or target_raw and target_raw.get("model_type") != "qwen3": + raise PlanCompileError("Megatron integration currently requires dense Qwen3 Target weights") runtime = _runtime_values(config, draft, source_context_parallel_size=source_context_parallel_size) try: - context_compiler = compile_offline_context_contract if offline else compile_sglang_context_contract + context_compiler = ( + compile_offline_context_contract if offline else + compile_megatron_context_contract if runtime["inference_engine_type"] == "megatron" + else compile_sglang_context_contract + ) context_contract = context_compiler( max_sequence_length=_select(config, "training.max_seq_length"), context_parallel_size=int( @@ -1960,7 +2016,8 @@ def compile_execution_plan( name="target_feature_engine", requirement="required", requested=_select(config, "inference.inference_engine_type"), - resolved="offline_replay" if offline else "sglang", + resolved=("offline_replay" if offline else + _select(config, "model.target_model_backend", "sglang")), ), CapabilityDecision( name="draft_trainer", @@ -1972,7 +2029,7 @@ def compile_execution_plan( name="training_backend", requirement="required", requested=_select(config, "training.train_backend"), - resolved="fsdp", + resolved=_select(config, "training.train_backend", "fsdp"), ), CapabilityDecision( name="static_multi_node_execution", diff --git a/aspec/ray/placement_group.py b/aspec/ray/placement_group.py index c945e3f..07624e0 100644 --- a/aspec/ray/placement_group.py +++ b/aspec/ray/placement_group.py @@ -559,7 +559,7 @@ def create_placement_groups(args, roles: set[str] | None = None): _ensure_ray_initialized() - if getattr(args, "inference_engine_type", None) == "sgl": + if getattr(args, "inference_engine_type", None) in {"sgl", "megatron"}: managed_local = (getattr(args, "target_service", None) or {}).get("mode") == "managed-local" if managed_local: live_nodes = [node for node in ray.nodes() if node.get("Alive")] diff --git a/aspec/ray/train_group.py b/aspec/ray/train_group.py index 1699f43..2faca83 100644 --- a/aspec/ray/train_group.py +++ b/aspec/ray/train_group.py @@ -298,6 +298,11 @@ def set_eval_queues( def _expand_queues_for_usp(self, queues): actor_count = len(self._actor_handlers) + if getattr(self.args, "train_backend", "fsdp") == "megatron": + tp = self.args.draft_tensor_parallel_size + if actor_count % tp or len(queues) != actor_count // tp: + raise ValueError("Megatron needs one input queue per data-parallel replica") + return [queues[rank // tp] for rank in range(actor_count)] if len(queues) == actor_count: return queues diff --git a/aspec/runtime_identity.py b/aspec/runtime_identity.py index 6c8461a..1553483 100644 --- a/aspec/runtime_identity.py +++ b/aspec/runtime_identity.py @@ -24,6 +24,16 @@ # Patched files and their native v0.5.21 request/capture/runner dependencies. # A bounded set catches divergent checkouts without hashing the entire repo. +MEGATRON_RUNTIME_RELATIVE_PATHS = ( + "package_info.py", + "models/gpt/gpt_model.py", + "tensor_parallel/layers.py", + "tensor_parallel/mappings.py", + "distributed/distributed_data_parallel.py", + "transformer/spec_utils.py", + "transformer/transformer_block.py", +) + SGLANG_RUNTIME_RELATIVE_PATHS = ( "kernels/ops/attention/triton_gdn_fused_proj.py", "kernels/ops/attention/fla/chunk_delta_h.py", diff --git a/aspec/train_entry.py b/aspec/train_entry.py index 0a3602d..ec4d623 100644 --- a/aspec/train_entry.py +++ b/aspec/train_entry.py @@ -302,9 +302,9 @@ def _validate_and_configure_dflash(args, draft_model_config) -> None: algo = "DSpark" if is_dspark else "DFlash" engine_type = getattr(args, "inference_engine_type", "sgl") - if engine_type not in {"sgl", "offline"}: + if engine_type not in {"sgl", "megatron", "offline"}: raise NotImplementedError( - f"{algo} currently supports inference_engine_type='sgl' or 'offline', got '{engine_type}'." + f"{algo} requires inference_engine_type=sgl, megatron or offline; got {engine_type!r}" ) if getattr(args, "defer_tokenization", False): raise NotImplementedError(f"{algo} does not support defer_tokenization=True.") @@ -489,7 +489,7 @@ def train_async_no_generation(args, evidence: RunEvidenceRecorder | None = None) timer.log_summary() except BaseException: - if args.inference_engine_type == "sgl": + if args.inference_engine_type in {"sgl", "megatron"}: rollback_unused_targets(inference_engines) raise diff --git a/aspec/training/checkpoint.py b/aspec/training/checkpoint.py index baea2b4..7fbfe7c 100644 --- a/aspec/training/checkpoint.py +++ b/aspec/training/checkpoint.py @@ -51,6 +51,8 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]: "training_num_gpus_per_node", "attention_backend", "draft_context_parallel_size", + "draft_tensor_parallel_size", + "train_backend", "sp_ulysses_size", "sp_ring_size", "fsdp_strategy", @@ -144,6 +146,11 @@ def _validate_exact_resume_topology(args: Any, metadata: dict[str, Any], *, chec if saved_topology is not None and not isinstance(saved_topology, dict): raise RuntimeError("Exact resume checkpoint has invalid training_topology metadata") current_topology = _logical_training_topology(args, current_world_size) + if ((saved_topology or {}).get("train_backend", "fsdp") + != current_topology.get("train_backend", "fsdp") + or (saved_topology or {}).get("draft_tensor_parallel_size", 1) + != current_topology.get("draft_tensor_parallel_size", 1)): + raise RuntimeError("Exact resume cannot change the training backend or Draft TP; use weights-only initialization") if getattr(args, "draft_algorithm", None) == "dspark": saved_objective = (saved_topology or {}).get("training_objective") or {} for side in ("target", "draft"): @@ -282,7 +289,8 @@ def _rng_state_path(checkpoint_dir: Path, rank: int) -> Path: return checkpoint_dir / "rng" / f"rank-{int(rank):05d}.pt" -def _save_rank_rng_state(checkpoint_dir: Path, *, rank: int, world_size: int) -> None: +def _save_rank_rng_state(checkpoint_dir: Path, *, rank: int, world_size: int, + backend_rng_state=None) -> None: path = _rng_state_path(checkpoint_dir, rank) temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") state = { @@ -291,6 +299,7 @@ def _save_rank_rng_state(checkpoint_dir: Path, *, rank: int, world_size: int) -> "world_size": int(world_size), "torch": torch.get_rng_state(), "cuda": torch.cuda.get_rng_state() if torch.cuda.is_available() else None, + **({"backend_rng": backend_rng_state} if backend_rng_state is not None else {}), } try: torch.save(state, temporary) @@ -397,10 +406,15 @@ def __init__(self, model): self.model = model def state_dict(self): + if callable(getattr(self.model, "_aspec_checkpoint_state", None)): + return {"model": self.model._aspec_checkpoint_state()} model_state_dict, _ = get_state_dict(self.model, optimizers=[]) return {"model": model_state_dict} def load_state_dict(self, state_dict): + if callable(getattr(self.model, "_aspec_load_checkpoint_state", None)): + self.model._aspec_load_checkpoint_state(state_dict["model"]) + return set_state_dict( self.model, optimizers=[], model_state_dict=state_dict["model"], optim_state_dict=None ) @@ -445,7 +459,8 @@ def _optimizer_load_template(self): master = self.optimizer.fp32_params[index] if len(parts) != 2 or field not in {"step", "exp_avg", "exp_avg_sq"}: raise ValueError(f"Unsupported BF16 Adam checkpoint state: {key}") - expected_shape = () if field == "step" else tuple(master.shape) + logical_master = self._checkpoint_tensor(index, master) + expected_shape = () if field == "step" else tuple(logical_master.shape) dtype = value.properties.dtype valid_dtype = ( dtype in {torch.float32, torch.float64} @@ -462,11 +477,24 @@ def _optimizer_load_template(self): self._load_template = {**template, "state": states} return self._load_template + def _checkpoint_tensor(self, index, tensor): + wrap = getattr(self.optimizer, "checkpoint_tensor", None) + return wrap(index, tensor) if wrap is not None else tensor + + def _checkpoint_optimizer_state(self, state): + if not hasattr(self.optimizer, "checkpoint_tensor"): + return state + return {**state, "state": { + index: {key: self._checkpoint_tensor(int(index), value) + if isinstance(value, torch.Tensor) else value for key, value in values.items()} + for index, values in state["state"].items() + }} + def state_dict(self): if self._is_bf16_optimizer: names = _fp32_param_names(self.optimizer) return { - "optim": ( + "optim": self._checkpoint_optimizer_state( self._optimizer_load_template() if self._load_path is not None else self.optimizer.state_dict() @@ -474,8 +502,8 @@ def state_dict(self): # Keyed by name: which parameters are trainable can differ between the run # that saved and the run that loads. "fp32_params": { - name: param.data - for name, param in zip(names, self.optimizer.fp32_params, strict=True) + name: self._checkpoint_tensor(index, param.data) + for index, (name, param) in enumerate(zip(names, self.optimizer.fp32_params, strict=True)) }, } _, optimizer_state_dict = get_state_dict(self.model, optimizers=[self.optimizer]) @@ -483,7 +511,16 @@ def state_dict(self): def load_state_dict(self, state_dict): if self._is_bf16_optimizer: - self.optimizer.load_state_dict(state_dict["optim"]) + from torch.distributed.tensor import DTensor + + optim = state_dict["optim"] + if hasattr(self.optimizer, "checkpoint_tensor"): + optim = {**optim, "state": { + index: {key: value.to_local() if isinstance(value, DTensor) else value + for key, value in values.items()} + for index, values in optim["state"].items() + }} + self.optimizer.load_state_dict(optim) saved = state_dict.get("fp32_params") if saved: names = _fp32_param_names(self.optimizer) @@ -498,7 +535,8 @@ def load_state_dict(self, state_dict): ) with torch.no_grad(): for name, master in zip(names, self.optimizer.fp32_params, strict=True): - master.data.copy_(saved[name]) + value = saved[name] + master.data.copy_(value.to_local() if isinstance(value, DTensor) else value) logger.info("Restored fp32 master params from checkpoint") return set_state_dict( @@ -1036,6 +1074,9 @@ def finalize_load(actor: Any, checkpoint_payload: dict[str, Any] | None) -> None rng_state, legacy=isinstance(rng_state, dict) and "schema_version" not in rng_state, ) + restore_backend_rng = getattr(actor, "_restore_backend_rng", None) + if restore_backend_rng is not None: + restore_backend_rng(rng_state.get("backend_rng")) torch.set_rng_state(rng_state["torch"]) if torch.cuda.is_available() and rng_state.get("cuda") is not None: cuda_state = rng_state["cuda"] @@ -1105,6 +1146,8 @@ def save(actor: Any, step: int) -> None: checkpoint_dir, rank=dist.get_rank(), world_size=dist.get_world_size(), + backend_rng_state=(actor._checkpoint_backend_rng() + if hasattr(actor, "_checkpoint_backend_rng") else None), ) if dist.get_rank() == 0: diff --git a/aspec/training/data_fetcher.py b/aspec/training/data_fetcher.py index d807116..2f535bf 100644 --- a/aspec/training/data_fetcher.py +++ b/aspec/training/data_fetcher.py @@ -139,8 +139,18 @@ def __init__( packing_enabled: bool = False, feature_affinity: str = "any", consumer_node_ip: str | None = None, + tensor_parallel_group=None, + tensor_parallel_control_group=None, ): self.ray_queue = ray_queue + if (tensor_parallel_group is None) != (tensor_parallel_control_group is None): + raise ValueError("TP input requires paired GPU-data and CPU-control groups") + self.tensor_parallel_group = tensor_parallel_group + self.tensor_parallel_control_group = tensor_parallel_control_group + self._tensor_parallel_device = (torch.cuda.current_device() + if tensor_parallel_group is not None else None) + self._tp_source_prefetch = None + self._tensor_parallel_samples = self._tensor_parallel_payload_bytes = 0 self.mooncake_store = mooncake_store self.device = device self.prefetch_factor = prefetch_factor @@ -186,11 +196,21 @@ def __init__( ): raise ValueError("packing requires non-USP training and positive max_seq_length") self._init_sp_context() + if tensor_parallel_group is not None and prefetch_factor > 0 and dist.get_rank(tensor_parallel_group) == 0: + from aspec.training.tp_source_prefetch import TPSourcePrefetch + + self._tp_source_prefetch = TPSourcePrefetch( + self._read_tp_source, lambda item: _release_p2p_receive_leases(item[1]), depth=prefetch_factor) def request_stop(self) -> None: """Cancel future queue reads without interrupting a native transfer.""" self._stop_requested.set() + def close(self): + self.request_stop() + if self._tp_source_prefetch is not None: + self._tp_source_prefetch.close() + def _next_queue_item(self): # Poll only the queue wait, not Mooncake itself: close must be able to # join an idle prefetch worker before its registered buffers are freed. @@ -540,8 +560,14 @@ def __iter__(self) -> Iterator[Dict[str, torch.Tensor]]: logger.debug(f"__iter__: waiting for item from ray_queue (yield_count={yield_count})") try: - item = self._next_queue_item() + if self.tensor_parallel_group is not None: + item, data = self._tensor_parallel_item() + else: + item = self._next_queue_item() + data = None except Exception as e: + if self.tensor_parallel_group is not None: + raise logger.warning(f"__iter__: Exception waiting for data: {e}, timeout={self.timeout}") break @@ -550,7 +576,8 @@ def __iter__(self) -> Iterator[Dict[str, torch.Tensor]]: break logger.debug(f"__iter__: got item, mooncake_key={item.mooncake_key}") - data = self._load_from_mooncake(item) + if self.tensor_parallel_group is None: + data = self._load_from_mooncake(item) neutralized_count = self._resolve_and_neutralize_loss_mask( data, item.mooncake_key, neutralized_count @@ -584,6 +611,86 @@ def __iter__(self) -> Iterator[Dict[str, torch.Tensor]]: logger.debug(f"__iter__: yielding batch {yield_count}, keys={list(data.keys())}") yield data + def _tensor_parallel_item(self): + with torch.cuda.device(self._tensor_parallel_device): + return self._tensor_parallel_item_on_device() + + def _read_tp_source(self): + # No collation, GPU clones or collectives run on this producer thread. + # P2P GET writes preallocated destination slots; leases bound prefetch. + with torch.cuda.device(self._tensor_parallel_device): + item = self._next_queue_item() + return item, self._load_from_mooncake(item) if item is not None else {} + + def _tensor_parallel_item_on_device(self): + """Read a source lease once, then broadcast its full CP1 payload over TP. + + Source release and work-ledger acknowledgement belong to the TP leader. + Source GET may be prefetched by the leader. Every TP broadcast executes + on the consumer thread between microbatches, in model collective order. + Its destination lease remains attached through the consumer microbatch. + """ + group = self.tensor_parallel_group + leader = dist.get_global_rank(group, 0) + is_leader = dist.get_rank(group) == 0 + control, data, leases = [None], None, [] + if is_leader: + try: + if self._tp_source_prefetch is not None: + item, data = self._tp_source_prefetch.next() + else: + item, data = self._read_tp_source() + leases = _take_p2p_receive_leases(data) + control[0] = { + "item": item, + "values": {k: v for k, v in data.items() if not isinstance(v, torch.Tensor)}, + "tensors": [(k, tuple(v.shape), v.dtype, v.device.type) + for k, v in data.items() if isinstance(v, torch.Tensor)], + } + except BaseException as error: + control[0] = {"error": f"{type(error).__name__}: {error}"} + try: + # Waiting for the next queue item must never launch a waiting GPU + # collective on followers: CUDA allocation/synchronization in the + # current model step could otherwise wait for undispatched input. + dist.broadcast_object_list(control, src=leader, group=self.tensor_parallel_control_group) + message = control[0] + if "error" in message: + raise RuntimeError("TP input leader failed: " + message["error"]) + item = message["item"] + if item is None: + return None, None + if not is_leader: + data = dict(message["values"]) + self._feature_contract_runtime.validate_metadata( + tensor_shapes=item.tensor_shapes, tensor_dtypes=item.tensor_dtypes, + metadata=item.metadata, + ) + for key, shape, dtype, original_device in message["tensors"]: + buffer = (data[key].to(device="cuda").contiguous() if is_leader + else torch.empty(shape, device="cuda", dtype=dtype)) + dist.broadcast(buffer, src=leader, group=group) + if not is_leader: + data[key] = buffer.cpu() if original_device == "cpu" else buffer + if not is_leader: + names = self._feature_contract_runtime.contract["tensors"] + self._validate_received_feature_tensors( + {name: data[name] for name in names}, expected_shapes=item.tensor_shapes, + expected_dtypes={name: data[name].dtype for name in names}, + ) + if leases: + data["_mooncake_p2p_receive_leases"] = leases + leases = [] + self._tensor_parallel_samples += 1 + self._tensor_parallel_payload_bytes += sum( + data[key].numel() * data[key].element_size() + for key, _shape, _dtype, _device in message["tensors"] + ) + return item, data + finally: + for lease in leases: + lease.release() + def _direct_cp_get_sharded_item( self, neutralized_count: int ) -> tuple[Dict[str, torch.Tensor] | None, int]: @@ -957,6 +1064,14 @@ def get_feature_contract_status(self) -> dict[str, Any]: status["data_admission"] = { "neutralized_rank_samples": self._neutralized_rank_samples, } + if getattr(self, "tensor_parallel_group", None) is not None: + status["tensor_parallel_input"] = { + "rank": dist.get_rank(self.tensor_parallel_group), + "size": dist.get_world_size(self.tensor_parallel_group), + "broadcast_samples": self._tensor_parallel_samples, + "broadcast_payload_bytes": self._tensor_parallel_payload_bytes, + "source_read_owner": dist.get_rank(self.tensor_parallel_group) == 0, + } get_transport_status = getattr(self.mooncake_store, "get_p2p_status", None) transport = ( get_transport_status() if callable(get_transport_status) else None @@ -1011,6 +1126,8 @@ def create_mooncake_dataloader( packing_enabled: bool = False, feature_affinity: str = "any", consumer_node_ip: str | None = None, + tensor_parallel_group=None, + tensor_parallel_control_group=None, ) -> DataLoader: """Create a DataLoader that fetches from mooncake via queue. @@ -1028,7 +1145,7 @@ def create_mooncake_dataloader( collator: Collator for padding and batching samples. device: Target device for tensors. batch_size: Number of samples per batch (= per_dp_rank_batch_size). - prefetch_factor: Unused, kept for API compatibility. + prefetch_factor: Source GET queue depth for TP input; unused by other paths. timeout: Timeout in seconds for waiting on queue. None means wait forever. assistant_header_ids: Token IDs for assistant header (for loss mask skip check). end_token_ids: Token IDs for end of turn (for loss mask skip check). @@ -1061,6 +1178,8 @@ def create_mooncake_dataloader( packing_enabled=packing_enabled, feature_affinity=feature_affinity, consumer_node_ip=consumer_node_ip, + tensor_parallel_group=tensor_parallel_group, + tensor_parallel_control_group=tensor_parallel_control_group, ) def dataloader_collator(samples): @@ -1128,6 +1247,8 @@ def __init__( packing_enabled: bool = False, feature_affinity: str = "any", consumer_node_ip: str | None = None, + tensor_parallel_group=None, + tensor_parallel_control_group=None, ): self.batch_size = batch_size self._dataloader = create_mooncake_dataloader( @@ -1154,6 +1275,8 @@ def __init__( packing_enabled=packing_enabled, feature_affinity=feature_affinity, consumer_node_ip=consumer_node_ip, + tensor_parallel_group=tensor_parallel_group, + tensor_parallel_control_group=tensor_parallel_control_group, ) def __iter__(self) -> Iterator[Dict[str, torch.Tensor]]: @@ -1165,6 +1288,9 @@ def get_feature_contract_status(self) -> dict[str, Any]: def request_stop(self) -> None: self._dataloader.dataset.request_stop() + def close(self): + self._dataloader.dataset.close() + class PrefetchedDataFetcher: """Wraps MooncakeDataFetcher with async pre-fetching. diff --git a/aspec/training/dflash_trainer.py b/aspec/training/dflash_trainer.py index fbd5a28..344318c 100644 --- a/aspec/training/dflash_trainer.py +++ b/aspec/training/dflash_trainer.py @@ -94,6 +94,22 @@ def __init__(self, args: Namespace): self.verifier_norm: Optional[torch.nn.Module] = None self._mtp_binding_validated = False + _optimizer_class = BF16Optimizer + + def _loads_initial_weights_here(self): + return dist.get_rank() == 0 + + def _wrap_training_model(self, model, draft_model): + full_state = model.state_dict() if dist.get_rank() == 0 else {} + model = apply_fsdp2( + model, mesh=self.dp_mesh, cpu_offload=self.fsdp_cpu_offload, + args=self.args, modules_to_shard=list(draft_model.layers), + ) + return fsdp2_load_full_state_dict( + model, full_state, self.dp_mesh, + cpu_offload=True if self.fsdp_cpu_offload else None, + ) + def init_model( self, draft_model_config, @@ -145,7 +161,7 @@ def init_model( draft_model = self._build_draft_model(config) conditioning = getattr(config, "mtp_conditioning", None) - if dist.get_rank() == 0: + if self._loads_initial_weights_here(): if conditioning: draft_model.initialize_mtp_conditioning(target_model_path) initial_path = getattr(self.args, "initial_draft_model_path", None) @@ -175,22 +191,7 @@ def init_model( dflash_model = self._build_training_wrapper(draft_model) - full_state = dflash_model.state_dict() if dist.get_rank() == 0 else {} - - dflash_model = apply_fsdp2( - dflash_model, - mesh=self.dp_mesh, - cpu_offload=self.fsdp_cpu_offload, - args=self.args, - modules_to_shard=list(draft_model.layers), - ) - - dflash_model = fsdp2_load_full_state_dict( - dflash_model, - full_state, - self.dp_mesh, - cpu_offload=True if self.fsdp_cpu_offload else None, - ) + dflash_model = self._wrap_training_model(dflash_model, draft_model) if getattr(self.args, "compile_model", False): logger.info("Compiling DFlash model with torch.compile (inductor backend)") @@ -215,7 +216,7 @@ def init_model( decay_style = getattr(self.args, "lr_decay_style", "cosine") warmup_ratio = getattr(self.args, "warmup_ratio", 0.1) - self.optimizer = BF16Optimizer( + self.optimizer = self._optimizer_class( self.draft_model, lr=self.args.learning_rate, weight_decay=getattr(self.args, "weight_decay", 0.0), @@ -274,7 +275,7 @@ def init_model( self.prof.on_init_end() - logger.info(f"[Rank {self.dp_rank}] DFlash model initialized with FSDP2") + logger.info("[DP rank %s] Draft initialized with %s", self.dp_rank, self.execution_backend) return 0 diff --git a/aspec/training/megatron_optimizer.py b/aspec/training/megatron_optimizer.py new file mode 100644 index 0000000..1117b0f --- /dev/null +++ b/aspec/training/megatron_optimizer.py @@ -0,0 +1,47 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Retain ASpec's BF16 master-AdamW semantics over Megatron TP/DDP buffers.""" + +import torch +import torch.distributed as dist + +from aspec.models.megatron_draft import checkpoint_tensor +from aspec.training.optimizer import BF16Optimizer + + +class MegatronBF16Optimizer(BF16Optimizer): + def _model_gradient(self, parameter): + return parameter.main_grad + + def checkpoint_tensor(self, index, value): + return checkpoint_tensor(value, self.model_params[index], self.model._aspec_tp_mesh) + + def _clip_gradients(self): + group = self.model._aspec_tp_group + # Replicated vocabulary/confidence/norm parameters contribute once; + # every distinct attention/FFN shard contributes to the global norm. + gradients = [master.grad for parameter, master in zip( + self.model_params, self.fp32_params, strict=True) + if master.grad is not None + and (getattr(parameter, "_aspec_tp_axis", None) is not None or group.rank() == 0)] + norm = torch.nn.utils.get_total_norm(gradients).square() + dist.all_reduce(norm, op=dist.ReduceOp.SUM, group=group) + norm.sqrt_() + coefficient = (self.max_grad_norm / (norm + 1e-6)).clamp(max=1.0) + if self._fuse_clip_scaling: + self.optimizer.grad_scale = coefficient.reciprocal() + else: + from aspec.models.ops.gradient_scaling import scale_gradients_ + + self.optimizer.grad_scale = None + scale_gradients_([p.grad for p in self.fp32_params if p.grad is not None], coefficient) + return norm + + def step(self, closure=None): + norm = super().step(closure) + self.model._aspec_ddp.zero_grad_buffer() + return norm + + def zero_grad(self, set_to_none=True): + super().zero_grad(set_to_none=set_to_none) + self.model._aspec_ddp.zero_grad_buffer() diff --git a/aspec/training/megatron_trainer.py b/aspec/training/megatron_trainer.py new file mode 100644 index 0000000..a3764f2 --- /dev/null +++ b/aspec/training/megatron_trainer.py @@ -0,0 +1,195 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""DSpark's retained training loop with upstream Megatron TP and DDP. + +Inspired by the public Megatron Core 0.20 custom-model DDP interface. DSpark +algorithm, batching, optimizer-step normalization and recovery remain ASpec-owned. +""" + +import weakref +from pathlib import Path + +import torch +import torch.distributed as dist +from torch.distributed.device_mesh import DeviceMesh +from torch.distributed.tensor import DTensor + +from aspec.models.megatron import initialize_parallelism +from aspec.models.megatron_draft import ( + draft_transformer_config, + load_model_checkpoint_state, + model_checkpoint_state, + partition_dspark, +) +from aspec.training.dspark_trainer import DSparkTrainer +from aspec.training.megatron_optimizer import MegatronBF16Optimizer + + +class MegatronDSparkTrainer(DSparkTrainer): + execution_backend = "megatron" + _optimizer_class = MegatronBF16Optimizer + + def _setup_device_mesh(self): + self.groups = initialize_parallelism(self.args.draft_tensor_parallel_size, + seed=getattr(self.args, "seed", 42)) + self.tp_size, self.tp_rank = self.groups.tp.size(), self.groups.tp.rank() + self.dp_size, self.dp_rank = self.groups.dp.size(), self.groups.dp.rank() + self.dp_group = self.groups.dp + self.cp_size, self.cp_rank, self.cp_group = 1, 0, None + self.cache_rank = dist.get_rank() + self.tp_mesh = DeviceMesh.from_group(self.groups.tp, "cuda") + self.dp_mesh = DeviceMesh.from_group(self.groups.dp, "cuda") + self.mesh = self.grad_sync_mesh = self.dp_mesh + # Only source GET is prefetched. Input collectives run on the model's + # consumer thread, in a consistent order across overlapping TP/DP groups. + for replica in range(self.dp_size): + ranks = list(range(replica * self.tp_size, (replica + 1) * self.tp_size)) + group = dist.new_group(ranks, backend="nccl") + control_group = dist.new_group(ranks, backend="gloo") + eval_group = dist.new_group(ranks, backend="nccl") + eval_control_group = dist.new_group(ranks, backend="gloo") + if replica == self.dp_rank: + self.input_tp_group = group + self.input_tp_control_group = control_group + self.eval_input_tp_group = eval_group + self.eval_input_tp_control_group = eval_control_group + + def _get_init_weight_context_manager(self): + return lambda: torch.device("cpu") + + def _loads_initial_weights_here(self): + return True + + def _build_training_wrapper(self, draft_model): + wrapper = super()._build_training_wrapper(draft_model) + sample = wrapper._sample_anchor_positions + wrapper._aspec_anchor_tp_broadcasts = 0 + + def shared_anchors(*args, **kwargs): + anchors, keep = sample(*args, **kwargs) + if self.tp_size > 1: + leader = dist.get_global_rank(self.groups.tp, 0) + dist.broadcast(anchors, src=leader, group=self.groups.tp) + dist.broadcast(keep, src=leader, group=self.groups.tp) + wrapper._aspec_anchor_tp_broadcasts += 1 + return anchors, keep + + wrapper._sample_anchor_positions = shared_anchors + return wrapper + + def _wrap_training_model(self, model, draft_model): + from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig + from megatron.core.tensor_parallel import ( + set_defaults_if_not_set_tensor_model_parallel_attributes, + ) + + config = draft_transformer_config(draft_model.config, self.tp_size) + partition_dspark(draft_model, config, self.groups, self.tp_mesh) + for parameter in model.parameters(): + set_defaults_if_not_set_tensor_model_parallel_attributes(parameter) + ddp = DistributedDataParallel( + config, DistributedDataParallelConfig( + grad_reduce_in_fp32=True, overlap_grad_reduce=True, + average_in_collective=True, use_distributed_optimizer=False), + model, pg_collection=self.groups, + ) + ddp.broadcast_params() + self._tp_partial_gradients = [p.main_grad for p in model.parameters() + if getattr(p, "_aspec_tp_partial_gradient", False)] + self._tp_partial_buffer = torch.empty( + sum(g.numel() for g in self._tp_partial_gradients), device="cuda", dtype=torch.float32, + ) + self._tp_partial_views = list(self._tp_partial_buffer.split( + [g.numel() for g in self._tp_partial_gradients])) + # A weak reference avoids making a second module parent / state-dict cycle. + object.__setattr__(draft_model, "_aspec_ddp", weakref.proxy(ddp)) + ddp._aspec_checkpoint_state = lambda: model_checkpoint_state(model, self.tp_mesh) + ddp._aspec_load_checkpoint_state = lambda state: load_model_checkpoint_state(model, state) + return ddp + + def _normalize_accumulated_gradients(self, local_denominator, accumulation_steps): + self.model.finish_grad_sync() + if self.tp_size > 1: + flattened = [g.view(-1) for g in self._tp_partial_gradients] + torch._foreach_copy_(self._tp_partial_views, flattened) + dist.all_reduce(self._tp_partial_buffer, group=self.groups.tp) + torch._foreach_copy_(flattened, self._tp_partial_views) + denominator = local_denominator.detach().clone() + dist.all_reduce(denominator, group=self.dp_group) + scale = torch.where( + denominator > 0, + self.dp_size * accumulation_steps / denominator.clamp_min(1e-6), + torch.zeros_like(denominator), + ) + from aspec.models.ops.gradient_scaling import scale_gradients_ + + scale_gradients_([p.main_grad for p in self.optimizer.model_params], scale) + return denominator + + def _checkpoint_backend_rng(self): + from megatron.core.tensor_parallel.random import get_cuda_rng_tracker + + return {"backend": "megatron", "cuda_tracker": get_cuda_rng_tracker().get_states()} + + def _restore_backend_rng(self, state): + from megatron.core.tensor_parallel.random import get_cuda_rng_tracker + + if not isinstance(state, dict) or state.get("backend") != "megatron": + raise RuntimeError("Exact Megatron resume requires the model-parallel RNG tracker") + saved = state.get("cuda_tracker") + if not isinstance(saved, dict) or set(saved) != set(get_cuda_rng_tracker().get_states()): + raise RuntimeError("Megatron RNG tracker streams differ from the checkpoint") + if any(not isinstance(value, torch.Tensor) or value.dtype != torch.uint8 + or value.device.type != "cpu" or value.ndim != 1 for value in saved.values()): + raise RuntimeError("Invalid Megatron RNG tracker state") + get_cuda_rng_tracker().set_states(saved) + + def pop_committed_work(self, optimizer_step): + local = super().pop_committed_work(optimizer_step) + replicas = [None] * self.tp_size + dist.all_gather_object(replicas, local, group=self.groups.tp) + if any(items != local for items in replicas): + raise RuntimeError("TP ranks completed different ASpec work identities") + return local if self.tp_rank == 0 else [] + + def _kernel_observation(self): + wrapper = getattr(self, "dflash", None) + if getattr(wrapper, "l1_loss_backend", None) is None: + return None + return { + "l1_backend": wrapper.l1_loss_backend, + "l1_kernel_invocations": dict(wrapper.l1_kernel_invocations), + "anchor_tp_broadcasts": wrapper._aspec_anchor_tp_broadcasts, + "qk_preparation_backends": [layer.self_attn.qk_prepare_backend + for layer in self.draft_model.layers], + "gradient_checkpointing_enabled": self.draft_model.gradient_checkpointing, + "optimizer_fused_clip_scaling": self.optimizer._fuse_clip_scaling, + "last_batch": {name.removesuffix("_observed"): value + for name, value in vars(wrapper).items() + if name.endswith("_observed") and type(value) in (int, float)}, + "attention_layers": list(self.draft_model.last_attention_observations), + } + + def get_feature_contract_status(self): + status = super().get_feature_contract_status() + if observation := self._kernel_observation(): + status["dspark_execution"] = observation + return status + + def save_draft_model_for_serving(self, output_dir): + error = None + try: + if self.dp_rank == 0: + state = model_checkpoint_state(self.draft_model, self.tp_mesh) + state = {name: (value.full_tensor() if isinstance(value, DTensor) else value).cpu() + for name, value in state.items()} + if self.tp_rank == 0: + root = Path(output_dir) + root.mkdir(parents=True, exist_ok=True) + self.draft_model.config.save_pretrained(root) + torch.save(state, root / "pytorch_model.bin") + except Exception as caught: + error = caught + message = self._synchronize_serving_error(error) + if message is not None: + raise RuntimeError("Megatron Draft export failed: " + message) diff --git a/aspec/training/optimizer.py b/aspec/training/optimizer.py index bb5a06e..929a414 100644 --- a/aspec/training/optimizer.py +++ b/aspec/training/optimizer.py @@ -85,6 +85,20 @@ def __init__( wsd_decay_style=wsd_decay_style, ) + def _model_gradient(self, parameter): + return parameter.grad + + def _clip_gradients(self): + if self._fuse_clip_scaling: + grad_norm = torch.nn.utils.get_total_norm( + [p.grad for p in self.fp32_params if p.grad is not None] + ) + coefficient = (self.max_grad_norm / (grad_norm + 1e-6)).clamp(max=1.0) + self.optimizer.grad_scale = coefficient.reciprocal() + return grad_norm + self.optimizer.grad_scale = None + return torch.nn.utils.clip_grad_norm_(self.fp32_params, self.max_grad_norm) + def step(self, closure=None): """Perform optimizer step with gradient clipping. @@ -98,27 +112,17 @@ def step(self, closure=None): grad_destinations = [] grad_sources = [] for p, mp, g in zip(self.model_params, self.fp32_params, self.fp32_grads): - if p.grad is not None: + gradient = self._model_gradient(p) + if gradient is not None: grad_destinations.append(g) - grad_sources.append(p.grad) + grad_sources.append(gradient) mp.grad = g else: mp.grad = None if grad_destinations: _copy_tensors_(grad_destinations, grad_sources) - if self._fuse_clip_scaling: - grad_norm = torch.nn.utils.get_total_norm( - [p.grad for p in self.fp32_params if p.grad is not None] - ) - # Fused AdamW already supports device-side gradient unscaling. - # Apply the same clipping coefficient there, avoiding a complete - # read/write pass over the FP32 gradient buffers on Hopper. - coefficient = (self.max_grad_norm / (grad_norm + 1e-6)).clamp(max=1.0) - self.optimizer.grad_scale = coefficient.reciprocal() - else: - self.optimizer.grad_scale = None - grad_norm = torch.nn.utils.clip_grad_norm_(self.fp32_params, self.max_grad_norm) + grad_norm = self._clip_gradients() # Fused AdamW consumes this device scalar through the same path used by # GradScaler. A nonzero value skips parameter, moment, weight-decay, and diff --git a/aspec/training/tp_source_prefetch.py b/aspec/training/tp_source_prefetch.py new file mode 100644 index 0000000..460aa55 --- /dev/null +++ b/aspec/training/tp_source_prefetch.py @@ -0,0 +1,63 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Bounded source GET prefetch; GPU collectives stay on the consumer thread.""" + +import queue +import threading + + +class TPSourcePrefetch: + def __init__(self, read, release, *, depth): + self.read, self.release = read, release + self.queue = queue.Queue(maxsize=depth) + self.stop = threading.Event() + self.thread = None + self.error = None + self.exhausted = False + + def _run(self): + try: + while not self.stop.is_set(): + item = self.read() + while not self.stop.is_set(): + try: + self.queue.put(item, timeout=0.1) + break + except queue.Full: + continue + else: + self.release(item) + return + if item[0] is None: + return + except BaseException as error: + self.error = error + + def next(self): + if self.exhausted: + return None, {} + if self.thread is None: + self.thread = threading.Thread(target=self._run, daemon=True, name="tp-source-get") + self.thread.start() + while not self.stop.is_set(): + if self.error is not None: + raise self.error + try: + item = self.queue.get(timeout=0.1) + self.exhausted = item[0] is None + return item + except queue.Empty: + continue + raise StopIteration + + def close(self): + self.stop.set() + if self.thread is not None: + self.thread.join(timeout=10) + if self.thread.is_alive(): + raise RuntimeError("TP source GET did not stop; retain backing storage until the read fence") + while True: + try: + self.release(self.queue.get_nowait()) + except queue.Empty: + return diff --git a/aspec/training/trainer.py b/aspec/training/trainer.py index adf4900..f4eb9d9 100644 --- a/aspec/training/trainer.py +++ b/aspec/training/trainer.py @@ -74,12 +74,15 @@ def _consumer_participant( *, dp_rank: int, sequence_rank: int | None = None, + tensor_rank: int = 0, ) -> str: """Build a stable, unique identity for one feature/storage consumer.""" participant = f"draft_{role}_dp_rank_{dp_rank}" if sequence_rank is not None: participant = f"{participant}_sequence_rank_{sequence_rank}" + if tensor_rank: + participant = f"{participant}_tensor_rank_{tensor_rank}" return participant @@ -140,6 +143,8 @@ class Trainer(abc.ABC): logic via ``init_model``, ``_train_step``, and ``_aggregate_metrics``. """ + execution_backend = "fsdp" + def __init__(self, args: Namespace): self.args = args @@ -328,6 +333,7 @@ def set_train_queue( ) prefetch_depth = getattr(self.args, "prefetch_depth", 0) + tp_input = getattr(self, "input_tp_group", None) is not None gpu_device = torch.cuda.current_device() # When prefetching, stage data on CPU to avoid GPU contention between @@ -340,6 +346,7 @@ def set_train_queue( collator=collator, device=fetch_device, batch_size=per_dp_rank_batch_size, + prefetch_factor=prefetch_depth if tp_input else 2, assistant_header_ids=self.assistant_header_ids, end_token_ids=self.end_token_ids, dynamic_loss_mask=self.dynamic_loss_mask, @@ -355,15 +362,18 @@ def set_train_queue( "trainer", dp_rank=self.dp_rank, sequence_rank=sequence_rank, + tensor_rank=getattr(self, "tp_rank", 0), ), lifecycle_observer=lifecycle_observer, packing_enabled=packing_enabled, feature_affinity=getattr( self.args, "mooncake_feature_affinity", "any" ), + tensor_parallel_group=getattr(self, "input_tp_group", None), + tensor_parallel_control_group=getattr(self, "input_tp_control_group", None), ) - if prefetch_depth > 0: + if prefetch_depth > 0 and not tp_input: self.data_fetcher = PrefetchedDataFetcher( inner_fetcher, prefetch_depth=prefetch_depth, @@ -372,12 +382,13 @@ def set_train_queue( logger.info( f"[Rank {self.dp_rank}] Prefetched data fetcher initialized " f"(batch_size={per_dp_rank_batch_size}, prefetch_depth={prefetch_depth}, " - f"staging=CPU, target={gpu_device})" + f"staging={'GPU TP' if getattr(self, 'input_tp_group', None) is not None else 'CPU'}, target={gpu_device})" ) else: self.data_fetcher = inner_fetcher logger.info( f"[Rank {self.dp_rank}] Data fetcher initialized with batch_size={per_dp_rank_batch_size}" + + (f", source_prefetch_depth={prefetch_depth}, TP broadcasts on consumer thread" if tp_input else "") ) # ------------------------------------------------------------------ @@ -412,6 +423,7 @@ def set_eval_queue( collator=collator, device=torch.cuda.current_device(), batch_size=per_dp_rank_batch_size, + prefetch_factor=0, assistant_header_ids=self.assistant_header_ids, end_token_ids=self.end_token_ids, dynamic_loss_mask=self.dynamic_loss_mask, @@ -427,8 +439,11 @@ def set_eval_queue( "eval", dp_rank=self.dp_rank, sequence_rank=sequence_rank, + tensor_rank=getattr(self, "tp_rank", 0), ), lifecycle_observer=lifecycle_observer, + tensor_parallel_group=getattr(self, "eval_input_tp_group", None), + tensor_parallel_control_group=getattr(self, "eval_input_tp_control_group", None), ) self._eval_collator = collator self._eval_cache: list[dict] = [] diff --git a/aspec/training/trainer_actor.py b/aspec/training/trainer_actor.py index 2587568..4c082d7 100644 --- a/aspec/training/trainer_actor.py +++ b/aspec/training/trainer_actor.py @@ -119,7 +119,14 @@ def init(self, args: Namespace, role: str, mooncake_config=None, with_ref: bool if draft_model_config is None and getattr(args, "draft_model_config", None): draft_model_config = AutoDraftModelConfig.from_file(args.draft_model_config) - trainer_class = _trainer_class_for_config(draft_model_config) + if getattr(args, "train_backend", "fsdp") == "megatron": + from aspec.training.megatron_trainer import MegatronDSparkTrainer + + if not isinstance(draft_model_config, DSparkConfig): + raise ValueError("Megatron Draft currently requires DSpark") + trainer_class = MegatronDSparkTrainer + else: + trainer_class = _trainer_class_for_config(draft_model_config) self._trainer = trainer_class(args) target_model_path = getattr(args, "target_model_path", None) @@ -186,7 +193,8 @@ def get_status(self) -> dict: import aspec from aspec.inference.target_runtime import code_identity, dependency_versions - return { + trainer = getattr(self, "_trainer", None) + result = { "rank": self._rank, "world_size": self._world_size, "node_ip": self.get_node_ip(), @@ -196,11 +204,26 @@ def get_status(self) -> dict: "assigned_gpu_ids": [int(float(value)) for value in ray.get_gpu_ids()], "local_gpu_id": int(os.environ["LOCAL_RANK"]), "initialized": hasattr(self, "_trainer"), + "backend": getattr(trainer, "execution_backend", None), + "tensor_parallel_size": getattr(trainer, "tp_size", 1), + "tensor_parallel_rank": getattr(trainer, "tp_rank", 0), + "data_parallel_rank": getattr(trainer, "dp_rank", None), + "data_parallel_size": getattr(trainer, "dp_size", None), "runtime": {**code_identity(), "python": sys.executable, "prefix": sys.prefix, "aspec": aspec.__file__, "ray": ray.__file__, "torch": torch.__version__, "cuda": torch.version.cuda, - "dependencies": dependency_versions("nvidia-cutlass-dsl", "quack-kernels")}, + "dependencies": dependency_versions("nvidia-cutlass-dsl", "quack-kernels", + "megatron-core", "transformer-engine")}, } + if getattr(trainer, "execution_backend", None) == "megatron": + from aspec.runtime_identity import ( + MEGATRON_RUNTIME_RELATIVE_PATHS, + module_runtime_identity, + ) + + result["runtime"]["backend_identity"] = module_runtime_identity( + "megatron.core", relative_paths=MEGATRON_RUNTIME_RELATIVE_PATHS) + return result def save_model(self, step: int, force_sync: bool = False) -> None: self._trainer.save_model(step, force_sync) diff --git a/aspec/utils/env.py b/aspec/utils/env.py index 9e00c0c..1d37846 100644 --- a/aspec/utils/env.py +++ b/aspec/utils/env.py @@ -21,6 +21,10 @@ # compatibility directory. Ray runtime_env workers do not reliably retain # the driver's process loader path, so forward the resolved value exactly. "LD_LIBRARY_PATH", + "CUDA_HOME", + "CUDNN_HOME", + "CUDNN_FRONTEND_CUDART_LIB_NAME", + "CUDA_DEVICE_MAX_CONNECTIONS", # Source snapshots and explicitly selected backend adapters must resolve to # the same code in the driver and every Ray actor. "PYTHONPATH", @@ -81,6 +85,9 @@ def get_aspec_env_vars(args=None) -> dict[str, str]: **{k: "1" for k in _RAY_NOSET_VISIBLE_DEVICES_KEYS}, } env.update({k: os.environ[k] for k in _ASPEC_ENV_KEYS if k in os.environ}) + if (getattr(args, "train_backend", None) == "megatron" + or getattr(args, "target_model_backend", None) == "megatron"): + env.setdefault("CUDA_DEVICE_MAX_CONNECTIONS", "1") if getattr(args, "cluster_mode", "single_node") == "static_multi_node": # Ray is started from each node's own venv/environment. A driver's # library/cache paths must not overwrite another node's local layout. diff --git a/aspec/utils/train_dump.py b/aspec/utils/train_dump.py index 5a19912..c8c42f1 100644 --- a/aspec/utils/train_dump.py +++ b/aspec/utils/train_dump.py @@ -43,8 +43,9 @@ def extract_gradients(model: nn.Module) -> Dict[str, torch.Tensor]: """ gradients = {} for name, param in model.named_parameters(): - if param.grad is not None: - gradients[name] = param.grad.detach().clone().cpu() + gradient = param.grad if param.grad is not None else getattr(param, "main_grad", None) + if gradient is not None: + gradients[name] = gradient.detach().clone().cpu() return gradients diff --git a/docs/README.md b/docs/README.md index 18ba362..1abb6fe 100644 --- a/docs/README.md +++ b/docs/README.md @@ -18,6 +18,7 @@ The [project roadmap](../ROADMAP.md) lists implemented features and planned exte | Task | Guide | | --- | --- | | Configure resources, batches and Plan | [Configuration](user-guide/configuration.md) | +| Select Megatron versions and isolated environments | [Megatron integration](user-guide/megatron.md) | | Choose DSpark, DFlash, EAGLE or native MTP | [Algorithms](user-guide/algorithms.md) | | Choose Dense/MoE/MLA, SWA/CSA/QSA or packing/CP | [Structures](user-guide/structures.md) | | Use Ling Targets | [Ling recipes](user-guide/ling.md) | diff --git a/docs/framework/REFERENCES.md b/docs/framework/REFERENCES.md index d61105b..82606a6 100644 --- a/docs/framework/REFERENCES.md +++ b/docs/framework/REFERENCES.md @@ -12,6 +12,8 @@ the retained source identities; applicable license and notice texts are distribu | [AngelSpec](https://github.com/Tencent/AngelSpec) `d3412bed231025ea85d35dbf5e0af44f1ac62a5b` | Thread-default reference retained in `aspec/utils/env.py` | | Retained [AngelSpec](https://github.com/Tencent/AngelSpec) snapshot `0966eb139d1c4054767e4a37bfc8a33db5989a72` | Concurrency, Direct-CP, Target-prefill, model-configuration and tensor-transfer regression cases; original source/license ancestry remains applicable | | [Megatron-LM](https://github.com/NVIDIA/Megatron-LM) and [FlashAttention](https://github.com/Dao-AILab/flash-attention) | Declared Direct CP, sparse metadata, output compaction and attention bridge derivations | +| Upstream [Megatron Core 0.20.0](https://github.com/NVIDIA/Megatron-LM/tree/685fb8bd970185e9ac3283fe32a5d6667bf4b268) | External, unmodified runtime through public model/spec, TP and DDP interfaces; independently authored ASpec Qwen3/DSpark adapters reuse the retained Target norm and training graph | +| [Megatron Bridge Qwen3 mapping](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/f46e89eafb935fb0c75209afdb4502322250d0c7/src/megatron/bridge/models/qwen/qwen3_bridge.py) | Public grouped-QKV/gated-MLP mapping reference; inspired ASpec's independent strict HF-to-Core loader, with explicit BF16 norm order and per-feature comparisons | | Official [SGLang](https://github.com/sgl-project/sglang) v0.5.21 `e00930c5489053f26d86b179cee0d087f846acbb` | Current Target runtime plus `aspec/compat/patches/sglang-v0.5.21.patch` | | [cuDNN Frontend](https://github.com/NVIDIA/cudnn-frontend) `cb8cec8a9884f77fa514767237502bef7b7ad9f0` | Source of the adapted ASpec-owned DSpark Q/K CuTeDSL specialization and selected Frost primitives | | cuDNN Frontend **1.31.0 wheel**, with the FlashAttention/CuTe source chain retained | Adapted SM90/SM100 joint GQA backward, pre/postprocess, layouts and host dispatch; [source paths, hashes and licenses](../../aspec/kernels/cutedsl/cudnn_gqa/README.md) | diff --git a/docs/releasing.md b/docs/releasing.md index 08f8116..551de73 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,7 +1,10 @@ # Releasing -ASpec publishes the Draft package `ling-aspec`, Target package `ling-aspec-target`, and -shared code package `ling-aspec-runtime` at the same version. Python imports and the CLI use `aspec`. +ASpec builds the Draft package `ling-aspec`, SGLang Target package `ling-aspec-target`, +Megatron environment package `ling-aspec-megatron`, and shared code package +`ling-aspec-runtime` at the same version. Python imports and the CLI use `aspec`. +The [Megatron guide](user-guide/megatron.md) describes its native build prerequisites +and independent environment profile. ## Versioning @@ -39,8 +42,8 @@ maintenance tools. `setup.py` collects resources from their canonical source loc In the release repository, configure a GitHub environment named `pypi`: - Add the environment secret `PYPI_API_TOKEN` with an account-wide PyPI token whose account can - publish all three distributions. This workflow uploads them together; a token scoped to one - project cannot authorize the other two. + publish all four distributions. This workflow uploads them together; a token scoped to one + project cannot authorize the others. - Allow deployments from `v*` tags. - Add required reviewers if releases need approval before upload. diff --git a/docs/support.md b/docs/support.md index 1c01a98..ae4d7ba 100644 --- a/docs/support.md +++ b/docs/support.md @@ -27,6 +27,10 @@ passing Plan validation establishes legality, not performance, convergence or do ## Execution and assets +- Experimental upstream Megatron Core Target/Draft adapters cover Qwen3-8B dense DSpark, + single-node CP1/PP1, Target TP1/TP2 and Draft TP1/TP2 with DP. The roles select backends + independently and use separate environments. See [scope and validation](user-guide/megatron.md) + for current evidence and numerical/replay boundaries; no Core fork is required. - Single-node online training supports required NVLink and explicitly selected Store/transport paths. With packing enabled, DSpark/DFlash full and sliding attention preserve document semantics with FA4 or native cuDNN flattened-token execution at CP1 and above. Selected-entry CSA/QSA/DSA diff --git a/docs/user-guide/installation.md b/docs/user-guide/installation.md index 85b1d1a..c83ad4d 100644 --- a/docs/user-guide/installation.md +++ b/docs/user-guide/installation.md @@ -5,6 +5,8 @@ GPUs and a CUDA 13-compatible driver. The Target package supplies its matching C the host still needs the C++20 compiler and native libraries listed below. See [native prerequisites](installation-advanced.md#requirements) for the compiler, glibc and RDMA library requirements, or use the [container](containers.md). +The experimental [Megatron profile](megatron.md) uses Python >=3.12, a separate environment +package and staged Transformer Engine build; follow that guide when choosing Megatron for either role. ## Install from PyPI diff --git a/docs/user-guide/megatron.md b/docs/user-guide/megatron.md new file mode 100644 index 0000000..d9c5216 --- /dev/null +++ b/docs/user-guide/megatron.md @@ -0,0 +1,253 @@ +# Megatron environments and execution boundary + +The Megatron adapter targets upstream **Megatron Core 0.20.0**, tag `core_v0.20.0`, +commit `685fb8bd970185e9ac3283fe32a5d6667bf4b268`. ASpec does not import an internal +Megatron fork or modify Core. The current integration is under validation; use the +checks below before treating a topology as accepted. + +The initial scope is dense Qwen3 Target prefill and dense, full-GQA DSpark Draft +training, BF16, MBS1, CP1/PP1 on one node. Target and Draft select their execution backends +independently. Shared Target, Megatron Draft THD packing, CP, Target PP, decode and MTP conditioning +are outside this adapter's current boundary and are rejected. + +## Dependency profiles + +| Distribution | Environment | +| --- | --- | +| `ling-aspec-runtime` | Shared ASpec code, CLI and service protocol | +| `ling-aspec` | Existing ASpec Draft dependencies | +| `ling-aspec-target` | SGLang Target dependencies | +| `ling-aspec-megatron` | Megatron dependencies for either role | + +Install each role in its own isolated venv. All ASpec distributions in a run use +the same release version. Install Megatron with the common runtime directly; +combining its profile with either default environment package creates unsupported +dependency mixtures. Backend imports remain lazy. + +Megatron requires Python >=3.12. Its profile pins Torch 2.13.0, CUDA toolkit 13.0.3, +Transformer Engine 2.20.2, FA4 b34, CUTLASS DSL 4.7.1 and Quack 0.6.5. The ordinary +Draft and SGLang profiles retain their independent versions. + +Transformer Engine's PyTorch extension must be built against the execution +environment's Torch ABI. An unconstrained build environment can select a different +Torch. Source setup installs the build prerequisites first, then builds with +`--no-build-isolation`, selecting the venv's CUDA 13 toolkit and NCCL/cuDNN headers +explicitly. Including all NVIDIA wheel headers can put the transport's CUDA 12 +runtime headers before CUDA 13; the setup helper avoids this mixture. + +From a source checkout: + +```bash +bash setup.sh --megatron --python python3.12 --venv /absolute/path/to/draft-venv +bash setup.sh --megatron --target --python python3.12 --venv /absolute/path/to/target-venv +``` + +For a built release directory, create a Python 3.12 venv, activate it, and install: + +```bash +pip install -r /path/to/release/runtime/megatron-build.txt +aspec_nvidia=$(python -c 'import pathlib,sysconfig; print(pathlib.Path(sysconfig.get_paths()["purelib"]) / "nvidia")') +CUDA_HOME="$aspec_nvidia/cu13" \ + NVTE_CUDA_INCLUDE_PATH="$aspec_nvidia/cu13/include" \ + CPLUS_INCLUDE_PATH="$aspec_nvidia/cudnn/include:$aspec_nvidia/nccl/include" \ + NVTE_BUILD_USE_NVIDIA_WHEELS=0 MAX_JOBS=4 pip install --no-build-isolation \ + --find-links /path/to/release ling-aspec-megatron +aspec doctor --backend megatron --gpu +``` + +This describes the new release artifacts, not an already published PyPI release. +Models, internal source trees, CUDA drivers and machine-specific compatibility +libraries are not included in ASpec packages. The host needs a compatible driver +and the native prerequisites in the [installation guide](installation-advanced.md). + +Each process prepares native-library paths from its own interpreter. The optional +local supervisor removes automatically selected Draft library paths before starting +Target. A separate Target service requires a service address; managed-local mode +also requires an explicit Target interpreter. Neither role imports the other's +Python dependency graph or transfers Python model objects across the boundary. + +## Target and Draft adapters + +Megatron Target constructs an upstream GPT model from public layer specs. The +Qwen3 adapter maps HF grouped Q/K/V and gate/up weights strictly, including TP +partitions. Its norm spec retains Qwen3's BF16 normalize/cast/weight order. +Changing that rounding order can change the intermediate residual features even +when the final normalized output is close. + +Target runs frozen prefill with requested post-layer residual features and the +final normalized hidden state. A public output processor omits vocabulary-logit +materialization. Capture writes directly into bounded transport storage. The +initial implementation executes one sequence at a time in each engine's TP world; +request batching does not imply a fused multi-sequence prefill kernel. + +Megatron Draft partitions attention heads and dense FFN channels using public +Core TP projections, and uses Core DDP gradient buffers with ASpec's FP32-master +AdamW update and schedule. The retained DSpark graph still owns attention/QK +kernels, block compaction, bounded projection caches, tiled/indexed L1, statistics, +selective recomputation and deferred metrics. Vocabulary, Markov and confidence +computations remain replicated across TP. This preserves the existing global +softmax/L1 semantics; vocabulary sharding is not a claimed capability. + +One TP leader prefetches source GETs into bounded, preallocated destination rings. +TP broadcasts and collation run on the consumer thread between microbatches, in +the same order as model collectives. Availability and tensor metadata use a CPU +Gloo group; no background reader launches NCCL work. Training and evaluation own +separate input groups. With source prefetch enabled, destination slots must cover +`prefetch_depth + 2` samples, including the active consumer and an in-progress GET. +The destination lease remains attached until the consumer finishes. Source release +occurs once per DP replica. Anchors/keep masks are shared from the TP leader; +commit collection checks matching work identity and reports it once per replica. + +Gradients are normalized by the complete optimizer window's supervision weight, +pooled across DP. TP-partial Q/K norm gradients are reduced once, and clipping +counts each replicated parameter once. Checkpoints describe logical TP shards and +restore model, FP32 masters, moments, schedule and both ASpec and Core RNG streams. +Exact resume requires the same Draft backend, TP topology and world size. Cross-backend weights-only +initialization is a separate stage. Native export gathers the logical Draft weights +for the existing strict loader. + +## First functional path + +Use the existing Draft venv and a separate Megatron Target venv. Select one GPU +for each role and a fresh output directory: + +```bash +source /absolute/path/to/draft-venv/bin/activate +CUDA_VISIBLE_DEVICES=0,1 aspec quickstart qwen3-8b \ + --target /absolute/path/to/Qwen3-8B --output outputs/megatron-target-001 \ + --managed-target /absolute/path/to/megatron-target-venv/bin/python \ + --set model.target_model_backend=megatron \ + --set inference.inference_engine_type=megatron +``` + +This uses Megatron Target with the existing FSDP Draft and exercises training, +fresh-process state recovery, further training, native export and strict loading. +For a Megatron Draft, activate its own Megatron venv and add +`--set training.train_backend=megatron --set mooncake.destination_gpu_ring_slots=3`. +To pair that Draft with SGLang, use the +SGLang Target interpreter and keep the default Target backend/engine. The existing +Quickstart's two-GPU budget covers TP1 for both roles. + +For TP2, use the ordinary training configuration with +`training.draft_tensor_parallel_size=2` and two Draft GPUs per DP replica. +Megatron Target independently uses `inference.megatron.tp_size=2` and +`inference.inference_num_gpus_per_engine=2`. Source ring capacity counts engines +and DP replicas, rather than counting every TP rank as a separate sample owner. + +These are functional checks on bundled conversations, not held-out quality or +throughput results. Checkpoint state and RNG can restore exactly while a later +backward pass differs because of kernel reduction order; future bitwise replay +is measured separately. + +## Validation commands + +Run on allocated GPUs with the Megatron venv activated: + +```bash +torchrun --standalone --nproc_per_node=2 -m tools.validation.megatron_target_alignment \ + --model /absolute/path/to/Qwen3-8B --sequence-length 4096 --output /tmp/target-tp2.json +torchrun --standalone --nproc_per_node=2 -m tools.validation.megatron_draft_alignment \ + --model /absolute/path/to/Qwen3-8B --tp 2 --output /tmp/draft-tp2-check \ + --attention-backend cudnn_flex_attention --reference-tp-projections +``` + +Use ASpec's environment bootstrap when invoking the tools directly: +`python -m aspec._environment /absolute/path/to/venv/bin/torchrun ...`. +The Target check compares actual HF features and verifies frozen parameters. +The Draft check uses synthetic fixed features with real frozen model bindings; +it compares loss, gradients, updates and checkpoint replay. It does not establish +online transport, committed-work recovery, throughput or model quality. Complete +online pairing and lifecycle gates remain separate acceptance requirements. + +The default Draft reference uses unpartitioned GEMMs. The optional independent +PyTorch TP reference retains BF16 partial-output rounding and groups each +projection's input-gradient sum before other consumers accumulate into that input. +It uses neither Core TP operators nor distributed communication. Raw comparisons +against unpartitioned GEMMs remain separate: BF16 TP rounding can produce small +gradient differences that Adam amplifies into larger update differences. The +tool also checks clipping/Adam with the same gradients and records checkpoint +load fidelity, in-memory replay and checkpoint replay independently. A failed +future-bitwise or unpartitioned-update gate remains a failed gate. + +To exercise the online 4K path with accumulation and evaluation, provide +pretokenized Parquet inputs containing `input_ids` and `loss_mask`, and use six +allocated GPUs for Draft TP2 x DP2 plus Target TP2: + +```bash +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5 python -m aspec._environment \ + /absolute/path/to/megatron-draft-venv/bin/python \ + -m tools.validation.megatron_pipeline_acceptance \ + --model /absolute/path/to/Qwen3-8B \ + --train-data /absolute/path/to/train.parquet \ + --eval-data /absolute/path/to/eval.parquet \ + --target-python /absolute/path/to/megatron-target-venv/bin/python \ + --target-tp 2 --draft-tp 2 --dp 2 --accumulation 2 \ + --sequence-length 4096 --anchors 512 --steps 4 \ + --output /tmp/megatron-online-check +``` + +This source-only validation entry uses the ordinary launcher and existing recipe +audits. It creates a fresh checkpoint, restarts the driver and workers, checks +loaded state/RNG and committed work, continues training, and exports an asset for +strict loading. Keep native compiler caches on a filesystem that supports the +toolchain's concurrent cache operations. + +## Recorded online checks + +The 4K diagnostic corpus uses 48 training work identities and eight separate +evaluation identities, each with 4096 pretokenized tokens and unequal supervision +masks. Text is repeated for a bounded functional check; it is not a quality corpus. +Both checks below ran four optimizer updates with 512 anchors, accumulation two, +evaluation, a fresh process restart after update two, checkpoint state/RNG/work +audits, native export and strict reloading: + +| Target | Draft | Result | +| --- | --- | --- | +| SGLang TP1 | Megatron TP1 x DP1 | Passed | +| Megatron TP2 | Megatron TP2 x DP2 | Passed; 16 unique training work identities committed after four updates | + +The successful runs drained source and destination leases and ended with zero +resident feature outputs. Earlier failed/canceled runs exercised reader-process +fencing before source teardown and also ended with zero resident outputs. This +does not certify driver hangs, host loss or network partitions. + +## Numerical validation status + +The following diagnostic results were obtained on SM90 with the dependency +versions above and a locally supplied Qwen3-8B checkpoint. Draft checks use the +full DSpark geometry (1,126,422,017 trainable parameters), two unequal-supervision +microbatches per update, two updates and the retained cuDNN attention path. +They are numerical diagnostics, not a quality or throughput benchmark. + +| Check | Observed result | +| --- | --- | +| Frozen Target, TP1/TP2, 128 and 4096 tokens, five residual captures plus final norm | Passed; worst feature relative L2 0.8787%, minimum cosine 0.999961 | +| Draft, 128 tokens/16 anchors, TP1 | Passed loss/gradient/update and future bitwise replay gates | +| Draft, 128 tokens/16 anchors, TP2, matched TP reference | Passed; worst update relative L2 0.24594%; future bitwise replay passed | +| Draft, 128 tokens/16 anchors, TP2 x DP2, matched TP reference | Passed; worst update relative L2 0.74774%; pooled denominators matched exactly; future bitwise replay passed | +| Draft, 4096 tokens/512 anchors, TP1 | Loss/gradient/update tolerances passed (worst update 2.77321%); future bitwise replay failed | +| Draft, 4096 tokens/512 anchors, TP2, matched TP reference | Worst update 3.01748% exceeds the unchanged 3% gate; future bitwise replay failed | + +The 4K check remains **unaccepted**. Loading model/master/moment/scheduler state +was exact in the TP2 diagnostic, but subsequent backward arithmetic differed. +An in-memory restore of the same tensors also differed, independently of +checkpoint deserialization. This establishes arithmetic nondeterminism in the +tested execution; it does not establish that all remaining numerical differences +have that cause. The same-gradient independent clipping/Adam oracle agreed to +relative update L2 below 3.1e-7 in the recorded checks. + +The unpartitioned BF16 reference also remains a failed TP2 update comparison +(approximately 9.7% on the first short-context update). The matched TP reference +explains rounding boundaries; passing it does not erase that result. Full 4K +update/future-replay acceptance and matched resident/online performance +measurements remain open before treating this adapter as generally validated. + +The online 4K check records actual Q/K preparation, L1 invocations, block counts, +projection-cache occupancy, checkpointing and fused optimizer clipping. Q/K +CuTeDSL, tiled FP32 L1 and fused clipping were observed. The short diagnostic +compacted 16 sampled blocks to 12 valid blocks. A single 4K row with 512 anchors +has one head chunk and does not exercise the retained multi-chunk projection +cache/head-recomputation optimization. Its THD/packing validation belongs to the +next integration stage; retaining the shared code path is not activation evidence. +Use `--gradient-checkpointing` on either Draft validation command to check optional +backbone recomputation separately from the default workload. diff --git a/requirements/megatron-build.txt b/requirements/megatron-build.txt new file mode 100644 index 0000000..1099fb4 --- /dev/null +++ b/requirements/megatron-build.txt @@ -0,0 +1,10 @@ +# Install before building native Megatron/TE extensions, in the same venv. +torch==2.13.0 +cuda-toolkit[cccl,nvcc]==13.0.3.0 +numpy<2.4 +setuptools>=80 +wheel +packaging>=24.2 +pybind11 +cmake +ninja diff --git a/requirements/megatron-v0.20.0.txt b/requirements/megatron-v0.20.0.txt new file mode 100644 index 0000000..d16a659 --- /dev/null +++ b/requirements/megatron-v0.20.0.txt @@ -0,0 +1,17 @@ +# Independent Megatron environment for either Target or Draft (Python >=3.12). +# Upstream core_v0.20.0: 685fb8bd970185e9ac3283fe32a5d6667bf4b268. +megatron-core==0.20.0 +transformer-engine[pytorch,core_cu13]==2.20.2 +torch==2.13.0 +torchvision==0.28.0 +transformers==5.12.1 +cuda-toolkit[cccl,nvcc]==13.0.3.0 +apache-tvm-ffi==0.1.11 +flash-attn-4==4.0.0b34 +nvidia-cutlass-dsl[cu13]==4.7.1 +quack-kernels[cu13]==0.6.5 +nvidia-cudnn-frontend==1.31.0 +einops==0.8.2 +cmake +ninja +numba diff --git a/requirements/sglang-v0.5.21.txt b/requirements/sglang-v0.5.21.txt index eb37893..7a43663 100644 --- a/requirements/sglang-v0.5.21.txt +++ b/requirements/sglang-v0.5.21.txt @@ -1,5 +1,7 @@ # Target-only environment: retain SGLang's declared dependency versions. sglang==0.5.21 +# Already required by SGLang; explicit pin admits this prerelease in uv too. +cuda-tile==1.6.0rc5 flash-attn-4==4.0.0b34 cuda-toolkit[nvcc]==13.0.3.0 # Mooncake's native wheel needs libcudart.so.12 alongside Torch's CUDA 13. diff --git a/setup.py b/setup.py index 62324ac..9c61c64 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,9 @@ def run(self): ROOT / "examples/data/sample_conversations.jsonl", ROOT / "tools/recipe_catalog.json", ROOT / "requirements/sglang-v0.5.21.txt", - ROOT / "requirements/draft.txt"] + ROOT / "requirements/draft.txt", + ROOT / "requirements/megatron-v0.20.0.txt", + ROOT / "requirements/megatron-build.txt"] for source in assets: destination = package / "resources" / source.relative_to(ROOT) destination.parent.mkdir(parents=True, exist_ok=True) diff --git a/setup.sh b/setup.sh index d65f0a1..45c66a8 100755 --- a/setup.sh +++ b/setup.sh @@ -5,6 +5,7 @@ venv="$repo_root/.venv" base_python="" options=() role=draft +backend=default explicit_venv=0 while [[ $# -gt 0 ]]; do case "$1" in @@ -13,10 +14,11 @@ while [[ $# -gt 0 ]]; do if [[ $1 == --venv ]]; then venv=$2; explicit_venv=1; else base_python=$2; fi shift 2 ;; --target) role=target; options+=("$1"); shift ;; + --megatron) backend=megatron; options+=("$1"); shift ;; --dev|--gpu) options+=("$1"); shift ;; --help) - echo 'Usage: bash setup.sh [--venv PATH] [--python EXECUTABLE] [--target] [--dev] [--gpu]' - echo 'Creates an isolated Linux Python >=3.11 venv and installs pinned dependencies.' + echo 'Usage: bash setup.sh [--venv PATH] [--python EXECUTABLE] [--target] [--megatron] [--dev] [--gpu]' + echo 'Creates an isolated Linux Python >=3.11 venv (>=3.12 for Megatron) and installs pinned dependencies.' echo 'Defaults to .venv. Existing venvs require this checkout owner.' echo 'Does not install host packages/drivers, download models, start training or clean clusters.' exit 0 ;; @@ -24,24 +26,28 @@ while [[ $# -gt 0 ]]; do esac done if [[ $role == target && $explicit_venv == 0 ]]; then venv="$repo_root/.venv-target"; fi +if [[ $backend == megatron && $explicit_venv == 0 ]]; then venv="$repo_root/.venv-megatron-$role"; fi [[ $(uname -s) == Linux && $(uname -m) == x86_64 ]] || { echo 'Use Linux x86-64; see docs/user-guide/installation.md.' >&2; exit 2; } [[ -z ${PYTHONPATH:-} && -z ${PYTHONHOME:-} ]] || { echo 'Unset PYTHONPATH and PYTHONHOME before setup.' >&2; exit 2; } +minimum_python=11 +if [[ $backend == megatron ]]; then minimum_python=12; fi if [[ -z $base_python ]]; then for candidate in python3.12 python3.11 python3; do - if command -v "$candidate" >/dev/null && "$candidate" -c 'import sys; sys.exit(sys.version_info[:2] < (3,11))'; then + if command -v "$candidate" >/dev/null && "$candidate" -c 'import sys; sys.exit(sys.version_info[:2] < (3, int(sys.argv[1])))' "$minimum_python"; then base_python=$candidate; break fi done fi -[[ -n $base_python ]] || { echo 'Install Python >=3.11 or pass --python EXECUTABLE.' >&2; exit 2; } -"$base_python" - <<'CHECK' +[[ -n $base_python ]] || { echo "Install Python >=3.$minimum_python or pass --python EXECUTABLE." >&2; exit 2; } +"$base_python" - "$minimum_python" <<'CHECK' import sys -if sys.version_info[:2] < (3, 11): - raise SystemExit("Python >=3.11 is required") +minimum = (3, int(sys.argv[1])) +if sys.version_info[:2] < minimum: + raise SystemExit(f"Python >=3.{minimum[1]} is required") CHECK venv=$("$base_python" -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$venv") if [[ -e $venv || -L $venv ]]; then diff --git a/tests/cpu/test_megatron.py b/tests/cpu/test_megatron.py new file mode 100644 index 0000000..4630450 --- /dev/null +++ b/tests/cpu/test_megatron.py @@ -0,0 +1,189 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +import subprocess +import sys +import threading +from argparse import Namespace +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch +import torch.nn.functional as F + +from aspec.config.megatron import validate_megatron_execution +from aspec.config.train_config import load_config +from aspec.feature_contract import build_target_feature_contract, validate_feature_contract +from aspec.models.megatron import qwen3_parameter +from aspec.planning import compile_execution_plan + + +def test_baseline_import_does_not_require_megatron(): + result = subprocess.run([sys.executable, "-c", """ +import sys +class RejectMegatron: + def find_spec(self, name, *args): + if name == 'megatron' or name.startswith('megatron.'): + raise AssertionError('Baseline imported an optional backend') +sys.meta_path.insert(0, RejectMegatron()) +import aspec.inference.factory +import aspec.training.trainer_actor +"""], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + +def settings(**overrides): + return Namespace(**{"train_backend": "megatron", "target_model_backend": "megatron", + "draft_tensor_parallel_size": 2, "world_size": 4, + "megatron_tp_size": 2, "inference_num_gpus_per_engine": 2, + "target_prefill_memory_optimized": True, **overrides}) + + +@pytest.mark.parametrize("target,training", [("megatron", "fsdp"), ("sglang", "megatron"), + ("megatron", "megatron")]) +def test_independent_target_draft_pairings(target, training): + validate_megatron_execution(settings(target_model_backend=target, train_backend=training, + draft_tensor_parallel_size=1 if training == "fsdp" else 2), + {"family": "dspark"}) + + +@pytest.mark.parametrize("overrides,message", [ + ({"world_size": 3}, "divisible"), + ({"draft_context_parallel_size": 2}, "CP1"), + ({"dflash_packing": True}, "packed"), + ({"fsdp_cpu_offload": True}, "offload"), + ({"distributed_backend": "gloo"}, "NCCL"), + ({"micro_batch_size": 2}, "micro_batch_size"), + ({"prefetch_depth": 1, "mooncake_destination_gpu_ring_slots": 1}, "destination_gpu_ring_slots"), + ({"sglang_pp_size": 2}, "PP1"), + ({"target_prefill_memory_optimized": False}, "memory_optimized"), + ({"inference_num_gpus_per_engine": 1}, "must equal"), + ({"train_backend": "fsdp", "draft_tensor_parallel_size": 2}, "Draft TP"), +]) +def test_megatron_fails_closed_before_process_launch(overrides, message): + with pytest.raises(ValueError, match=message): + validate_megatron_execution(settings(**overrides), {"family": "dspark"}) + + +def test_producer_backend_is_part_of_the_feature_identity(): + kwargs = dict(drafter_family="dspark", target_hidden_size=16, target_num_hidden_layers=4, + aux_hidden_state_layers=[0, 2], store_last_hidden_states=True) + sglang = build_target_feature_contract(**kwargs) + megatron = build_target_feature_contract(**kwargs, producer_backend="megatron") + validate_feature_contract(megatron) + assert sglang["contract_id"] != megatron["contract_id"] + + +def test_tp_source_capacity_counts_dp_replicas_and_target_engines(): + recipe = Path(__file__).resolve().parents[2] / "configs/recipes/quickstart_qwen3_8b.yaml" + overrides = [ + "model.target_model_backend=megatron", "inference.inference_engine_type=megatron", + "training.train_backend=megatron", "training.draft_tensor_parallel_size=2", + "training.training_num_gpus_per_node=4", "training.draft_accumulation_steps=1", + "inference.megatron.tp_size=2", "inference.inference_num_gpus=2", + "inference.inference_num_gpus_per_engine=2", "inference.inference_batch_size=1", + "mooncake.source_gpu_ring_slots=2", + "mooncake.destination_gpu_ring_slots=3", + ] + config = load_config(str(recipe), cli_args=overrides) + plan = compile_execution_plan(config) + assert plan.to_runtime_args().dp_size == 2 + assert plan.policy.pipeline["global_batch_size"] == 2 + # Two Target GPUs still represent one engine with one source ring. + with pytest.raises(ValueError, match="cannot retain one complete Draft dispatch"): + load_config(base_config=config, cli_args=["mooncake.source_gpu_ring_slots=1"]) + + +def test_tp_receive_slot_remains_owned_until_consumer_releases_it(monkeypatch): + from aspec.training.data_fetcher import MooncakeDataset, _release_p2p_receive_leases + + dataset = object.__new__(MooncakeDataset) + dataset.tensor_parallel_group = object() + dataset.tensor_parallel_control_group = object() + dataset._tp_source_prefetch = None + dataset._tensor_parallel_samples = dataset._tensor_parallel_payload_bytes = 0 + dataset._next_queue_item = lambda: "sample" + receive_buffer = torch.zeros(3) + ring = {"busy": False, "reads": 0, "releases": 0} + + def release(): + assert ring["busy"] + ring["busy"] = False + ring["releases"] += 1 + + def load(_item): + assert not ring["busy"], "next prefetch cannot wait on the active consumer" + ring["busy"] = True + ring["reads"] += 1 + receive_buffer.fill_(ring["reads"]) + return {"features": receive_buffer, + "_mooncake_p2p_receive_lease": SimpleNamespace(release=release)} + + dataset._load_from_mooncake = load + dataset._read_tp_source = lambda: ("sample", load("sample")) + monkeypatch.setattr(torch.distributed, "get_global_rank", lambda *_: 0) + monkeypatch.setattr(torch.distributed, "get_rank", lambda *_: 0) + def broadcast_control(*_, group, **__): + assert group is dataset.tensor_parallel_control_group + + monkeypatch.setattr(torch.distributed, "broadcast_object_list", broadcast_control) + monkeypatch.setattr(torch.distributed, "broadcast", lambda *_, **__: None) + original_to = torch.Tensor.to + + def to(tensor, *args, **kwargs): + return tensor if kwargs.get("device") == "cuda" else original_to(tensor, *args, **kwargs) + + monkeypatch.setattr(torch.Tensor, "to", to) + _, first = dataset._tensor_parallel_item_on_device() + assert first["features"].tolist() == [1, 1, 1] + assert ring["busy"] and ring["releases"] == 0 + _release_p2p_receive_leases(first) + _, second = dataset._tensor_parallel_item_on_device() + assert second["features"].tolist() == [2, 2, 2] + _release_p2p_receive_leases(second) + assert ring == {"busy": False, "reads": 2, "releases": 2} + + +def test_grouped_qkv_tp_mapping_preserves_independent_hf_projections(): + generator = torch.Generator().manual_seed(1729) + tensors = {f"model.layers.0.self_attn.{name}_proj.weight": torch.randn(rows, 16, generator=generator) + for name, rows in (("q", 16), ("k", 8), ("v", 8))} + reader = SimpleNamespace(tensor=lambda key: tensors[key]) + config = SimpleNamespace(num_attention_heads=4, num_key_value_heads=2, hidden_size=16, head_dim=4) + inputs = torch.randn(3, 16, generator=generator) + queries, keys, values = [], [], [] + for rank in range(2): + mapped = qwen3_parameter(reader, "decoder.layers.0.self_attention.linear_qkv.weight", + config, rank, 2) + projected = F.linear(inputs, mapped).view(3, 1, 4, 4) + queries.append(projected[:, :, :2].flatten(1)) + keys.append(projected[:, :, 2].flatten(1)) + values.append(projected[:, :, 3].flatten(1)) + for name, shards in (("q", queries), ("k", keys), ("v", values)): + torch.testing.assert_close(torch.cat(shards, dim=-1), + F.linear(inputs, tensors[f"model.layers.0.self_attn.{name}_proj.weight"])) + + +def test_source_prefetch_only_reads_in_background_and_drains_unused_leases(): + from aspec.training.tp_source_prefetch import TPSourcePrefetch + + main_thread = threading.get_ident() + calls, released = [], [] + second_read = threading.Event() + values = iter((("one", {"id": 1}), ("two", {"id": 2}), (None, {}))) + + def read(): + calls.append(threading.get_ident()) + item = next(values) + if item[0] == "two": + second_read.set() + return item + + prefetch = TPSourcePrefetch(read, lambda item: released.append(item[1]), depth=1) + assert prefetch.next() == ("one", {"id": 1}) + assert second_read.wait(timeout=2) + prefetch.close() + assert calls and all(thread != main_thread for thread in calls) + assert all(item.get("id") in (None, 2) for item in released) + assert {"id": 2} in released diff --git a/tests/cpu/test_release_audit.py b/tests/cpu/test_release_audit.py index 0d79272..30ad450 100644 --- a/tests/cpu/test_release_audit.py +++ b/tests/cpu/test_release_audit.py @@ -61,7 +61,7 @@ def _artifacts( link.type = tarfile.SYMTYPE link.linkname = "target" output.addfile(link) - for role in ("ling-aspec", "ling-aspec-target"): + for role in ("ling-aspec", "ling-aspec-target", "ling-aspec-megatron"): normalized = role.replace("-", "_") metadata = (f"Name: {role}\nVersion: {wheel_version}\n" f"Requires-Dist: ling-aspec-runtime=={wheel_version}\n") @@ -81,7 +81,7 @@ def test_release_audit_accepts_complete_artifact_pair(tmp_path): result = audit(tmp_path) assert result[wheel.name] == len(WHEEL_REQUIRED) + 1 assert result[source.name] == len(SDIST_REQUIRED) - assert len(result) == 6 + assert len(result) == 8 @pytest.mark.parametrize("member", ["aspec/compat/sglang_patch.py", @@ -119,7 +119,7 @@ def test_release_audit_rejects_non_source_members(tmp_path, member): def test_release_audit_requires_one_wheel_and_one_sdist(tmp_path): - with pytest.raises(ValueError, match="exactly three wheels and three sdists"): + with pytest.raises(ValueError, match="exactly four wheels and four sdists"): audit(tmp_path) diff --git a/tests/cpu/test_target_service.py b/tests/cpu/test_target_service.py index ddcb8d5..d790327 100644 --- a/tests/cpu/test_target_service.py +++ b/tests/cpu/test_target_service.py @@ -57,7 +57,7 @@ def register(sig, handler): monkeypatch.setattr(module, "TargetService", lambda **_: instance) monkeypatch.setattr(module, "http_server", lambda *_: server) monkeypatch.setattr(module, "visible_gpu_ids", lambda: [0]) - monkeypatch.setattr(module, "inventory", lambda: {"python": sys.executable}) + monkeypatch.setattr(module, "inventory", lambda backend="sglang": {"python": sys.executable}) monkeypatch.setattr(module, "ray", MagicMock()) monkeypatch.setattr(module.tempfile, "mkdtemp", lambda **_: str(tmp_path)) monkeypatch.setattr(module.time, "sleep", lambda seconds: events.append(("wait", seconds))) diff --git a/tools/audit_release.py b/tools/audit_release.py index 663a75d..44fde18 100644 --- a/tools/audit_release.py +++ b/tools/audit_release.py @@ -75,6 +75,8 @@ "aspec/compat/patches/sglang-v0.5.21.patch", "requirements/sglang-v0.5.21.txt", "requirements/draft.txt", + "requirements/megatron-v0.20.0.txt", + "requirements/megatron-build.txt", "tools/audit_release.py", "tools/recipe_catalog.json", "tools/build_release.py", @@ -107,9 +109,18 @@ "aspec/commands/convert_pretokenized_jsonl.py", "aspec/commands/_recipe_smoke.py", "aspec/commands/_checkpoint_audit.py", + "aspec/config/megatron.py", + "aspec/models/megatron.py", + "aspec/models/megatron_draft.py", + "aspec/training/megatron_trainer.py", + "aspec/training/megatron_optimizer.py", + "aspec/training/tp_source_prefetch.py", + "aspec/inference/engine/megatron_engine.py", "aspec/commands/_recipe_capacity.py", "aspec/resources/requirements/sglang-v0.5.21.txt", "aspec/resources/requirements/draft.txt", + "aspec/resources/requirements/megatron-v0.20.0.txt", + "aspec/resources/requirements/megatron-build.txt", "aspec/resources/tools/recipe_catalog.json", "aspec/resources/examples/data/sample_conversations.jsonl", } @@ -146,7 +157,7 @@ def _identity(raw: bytes, artifact: Path) -> tuple[str, str]: version = metadata.get("Version") if not name or not version: raise ValueError(f"{artifact.name}: package metadata lacks Name or Version") - if name not in {"ling-aspec", "ling-aspec-target", "ling-aspec-runtime"}: + if name not in {"ling-aspec", "ling-aspec-target", "ling-aspec-runtime", "ling-aspec-megatron"}: raise ValueError(f"{artifact.name}: unexpected distribution name {name!r}") if name != "ling-aspec-runtime": requirements = {value.replace(" ", "") for value in metadata.get_all("Requires-Dist", [])} @@ -198,7 +209,7 @@ def _sdist_members(path: Path) -> tuple[set[str], tuple[str, str]]: missing = SDIST_REQUIRED - names if root.startswith("ling_aspec_runtime-") else {"PKG-INFO", "LICENSE", "README.md", "pyproject.toml"} - names if missing: raise ValueError(f"{path.name}: missing source members {sorted(missing)}") - if not root.startswith(("ling_aspec-", "ling_aspec_target-", "ling_aspec_runtime-")): + if not root.startswith(("ling_aspec-", "ling_aspec_target-", "ling_aspec_runtime-", "ling_aspec_megatron-")): raise ValueError(f"{path.name}: unexpected source root {root!r}") with tarfile.open(path, mode="r:gz") as source: metadata_member = source.getmember(f"{root}/PKG-INFO") @@ -212,19 +223,19 @@ def _sdist_members(path: Path) -> tuple[set[str], tuple[str, str]]: def audit(directory: Path) -> dict[str, int]: wheels = sorted(directory.glob("*.whl")) sdists = sorted(directory.glob("*.tar.gz")) - if len(wheels) != 3 or len(sdists) != 3: - raise ValueError(f"{directory}: expected exactly three wheels and three sdists") + if len(wheels) != 4 or len(sdists) != 4: + raise ValueError(f"{directory}: expected exactly four wheels and four sdists") result, identities = {}, {} for artifact in [*wheels, *sdists]: members, identity = (_wheel_members(artifact) if artifact.suffix == ".whl" else _sdist_members(artifact)) result[artifact.name] = len(members) identities.setdefault(identity, []).append(artifact.suffix == ".whl") - expected = {"ling-aspec", "ling-aspec-target", "ling-aspec-runtime"} + expected = {"ling-aspec", "ling-aspec-target", "ling-aspec-runtime", "ling-aspec-megatron"} if ({name for name, version in identities} != expected or len({version for name, version in identities}) != 1 or any(sorted(kinds) != [False, True] for kinds in identities.values())): - raise ValueError("wheel identity does not match the three version-aligned source identities") + raise ValueError("wheel identity does not match the four version-aligned source identities") return result diff --git a/tools/build_release.py b/tools/build_release.py index ee00aef..befd036 100644 --- a/tools/build_release.py +++ b/tools/build_release.py @@ -33,10 +33,13 @@ def build_release(directory): subprocess.run([sys.executable, "-m", "build", "--outdir", str(directory)], cwd=root, check=True) project = tomllib.loads((root / "pyproject.toml").read_text())["project"] - for role, manifest in (("draft", "draft.txt"), ("target", "sglang-v0.5.21.txt")): + for role, manifest, name, python_min in ( + ("draft", "draft.txt", "ling-aspec", "3.11"), + ("target", "sglang-v0.5.21.txt", "ling-aspec-target", "3.11"), + ("megatron", "megatron-v0.20.0.txt", "ling-aspec-megatron", "3.12"), + ): with tempfile.TemporaryDirectory(prefix="aspec-role-") as staging: stage = Path(staging) - name = "ling-aspec" if role == "draft" else "ling-aspec-target" dependencies = [f"ling-aspec-runtime=={project['version']}"] dependencies += [line.strip() for line in (root / "requirements" / manifest).read_text().splitlines() if line.strip() and not line.lstrip().startswith("#")] @@ -45,7 +48,7 @@ def build_release(directory): f"name = {json.dumps(name)}\nversion = {json.dumps(project['version'])}\n" f"description = {json.dumps('ASpec ' + role + ' environment')}\n" "readme = \"README.md\"\nlicense = \"Apache-2.0\"\n" - "license-files = [\"LICENSE\"]\nrequires-python = \">=3.11\"\n" + f"license-files = [\"LICENSE\"]\nrequires-python = \">={python_min}\"\n" f"dependencies = {json.dumps(dependencies)}\n\n" "[project.urls]\nHomepage = \"https://github.com/inclusionAI/ASpec\"\n" "\n[tool.setuptools]\npackages = []\n") @@ -59,7 +62,7 @@ def build_release(directory): print(f"PASS {artifact}: {count} reviewed members", flush=True) runtime = directory / "runtime" runtime.mkdir() - for name in ("sglang-v0.5.21.txt", "draft.txt"): + for name in ("sglang-v0.5.21.txt", "draft.txt", "megatron-v0.20.0.txt", "megatron-build.txt"): shutil.copyfile(root / "requirements" / name, runtime / name) files = sorted(path for path in directory.rglob("*") if path.is_file()) (directory / "SHA256SUMS").write_text("".join( diff --git a/tools/install.sh b/tools/install.sh index 0a088ab..d030c4e 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -2,14 +2,16 @@ set -euo pipefail dev=0 role=draft +backend=default gpu=() for argument in "$@"; do case "$argument" in --dev) dev=1 ;; --target) role=target ;; + --megatron) backend=megatron ;; --gpu) gpu=(--gpu) ;; --help) - echo 'Usage: bash tools/install.sh [--target] [--dev] [--gpu]' + echo 'Usage: bash tools/install.sh [--target] [--megatron] [--dev] [--gpu]' echo 'For an already activated isolated venv. New users: bash setup.sh' exit 0 ;; *) printf 'Unknown argument: %s\n' "$argument" >&2; exit 2 ;; @@ -48,9 +50,27 @@ if [[ $role == target ]]; then fi extras="." (( dev == 0 )) || extras=".[dev]" -python -m pip install -r "$manifest" -e "$extras" +if [[ $backend == megatron ]]; then + python -c 'import sys; sys.exit("Megatron requires Python >=3.12" if sys.version_info[:2] < (3,12) else 0)' + manifest=requirements/megatron-v0.20.0.txt + check+=(--backend megatron) + # Build TE against the actual execution Torch, including its native headers. + # An unconstrained isolated build can select a different Torch ABI. + python -m pip install -r requirements/megatron-build.txt + aspec_nvidia=$(python -c 'import pathlib,sysconfig; print(pathlib.Path(sysconfig.get_paths()["purelib"]) / "nvidia")') + # The transport also needs CUDA 12's runtime. Including every NVIDIA wheel + # would put its headers ahead of CUDA 13 and break TE's C++ compilation. + CUDA_HOME="$aspec_nvidia/cu13" \ + NVTE_CUDA_INCLUDE_PATH="$aspec_nvidia/cu13/include" \ + CPLUS_INCLUDE_PATH="$aspec_nvidia/cudnn/include:$aspec_nvidia/nccl/include:${CPLUS_INCLUDE_PATH:-}" \ + NVTE_BUILD_USE_NVIDIA_WHEELS=0 MAX_JOBS="${MAX_JOBS:-4}" \ + python -m pip install --no-build-isolation -r "$manifest" -e "$extras" + printf 'megatron\n' > "$VIRTUAL_ENV/.aspec-backend" +else + python -m pip install -r "$manifest" -e "$extras" +fi printf '%s\n' "$role" > "$VIRTUAL_ENV/.aspec-role" -if [[ $role == target ]]; then +if [[ $role == target && $backend != megatron ]]; then aspec patch-sglang aspec patch-sglang --check fi diff --git a/tools/validation/megatron_draft_alignment.py b/tools/validation/megatron_draft_alignment.py new file mode 100644 index 0000000..3c0e6f8 --- /dev/null +++ b/tools/validation/megatron_draft_alignment.py @@ -0,0 +1,439 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Fixed-feature DSpark TP/DP loss, gradient, update and checkpoint checks. + +Uses the full Qwen3 DSpark geometry and real frozen embedding/head bindings. +Synthetic features isolate execution correctness; this is neither a quality +evaluation nor a throughput benchmark. The reference keeps ordinary ASpec +projections and uses an independent FP32-master PyTorch AdamW update. +""" + +import argparse +import copy +import json +import os +from contextlib import nullcontext +from pathlib import Path + +import torch +import torch.distributed as dist +import torch.distributed.checkpoint as dcp +import torch.nn.functional as F +from torch.distributed.tensor import DTensor + +from aspec.config import load_config +from aspec.models.draft.auto import AutoDraftModelConfig +from aspec.models.draft.dspark import DSparkDraftModel +from aspec.models.megatron_draft import checkpoint_tensor, model_checkpoint_state +from aspec.planning import compile_execution_plan +from aspec.training.checkpoint import LRSchedulerState, ModelState, OptimizerState +from aspec.training.dspark_trainer import DSparkTrainer +from aspec.training.lr_scheduler import LRSchedulerWithWarmup +from aspec.training.megatron_trainer import MegatronDSparkTrainer +from aspec.utils.distributed import init_direct_cp_groups, init_gloo_group + + +def fixture(config, *, length, anchors, step, micro, replica): + generator = torch.Generator().manual_seed(7100 + step * 101 + micro * 7 + replica * 31) + hidden, vocab = config.hidden_size, config.vocab_size + ids = torch.randint(vocab, (1, length), generator=generator, dtype=torch.int64) + features = torch.randn(1, length, config.num_target_layers * hidden, + generator=generator, dtype=torch.bfloat16) + teacher = torch.randn(1, length, hidden, generator=generator, dtype=torch.bfloat16) + mask = torch.ones(1, length) + prefix, tail = 17 + 11 * micro, 7 * (micro + 1) + mask[:, :prefix] = 0 + mask[:, -tail:] = 0 + positions = torch.linspace(prefix + 1, length - tail - 9, anchors).long().unsqueeze(0) + keep = torch.ones_like(positions, dtype=torch.bool) + if micro: + keep[:, -max(1, anchors // 4):] = False + positions = torch.where(keep, positions, torch.zeros_like(positions)) + return {"input_ids": ids.cuda(), "hidden_states_list": features.cuda(), + "last_hidden_states": teacher.cuda(), "loss_mask": mask.cuda(), + "injected_anchors": positions.cuda(), + "injected_keep_mask": keep.cuda()} + + +def reference_tp_projections(draft, size): + """Simulate TP GEMMs/reduction with ordinary single-process PyTorch. + + BF16 row partials round before the TP sum; an unpartitioned GEMM rounds + once. This independent oracle preserves those boundaries without using + Core TP operators or distributed communication. The unpartitioned reference + remains separately available; neither reference changes the native graph. + """ + class Column(torch.autograd.Function): + @staticmethod + def forward(ctx, value, weight): + ctx.save_for_backward(value, weight) + return torch.cat([F.linear(value, part.contiguous()) + for part in weight.chunk(size, dim=0)], dim=-1) + + @staticmethod + def backward(ctx, gradient): + value, weight = ctx.saved_tensors + inputs = value.reshape(-1, value.shape[-1]) + input_gradient, weight_gradients = None, [] + for grad, part in zip(gradient.chunk(size, dim=-1), weight.chunk(size, dim=0), strict=True): + grad = grad.contiguous().reshape(-1, grad.shape[-1]) + partial = grad @ part.contiguous() + input_gradient = partial if input_gradient is None else input_gradient + partial + weight_gradients.append(grad.T @ inputs) + # Group the BF16 dX sum inside each projection, before gradients + # from Q/K/V or other consumers meet at their common input. + return input_gradient.reshape_as(value), torch.cat(weight_gradients, dim=0) + + class Projection(torch.nn.Module): + def __init__(self, original, axis): + super().__init__() + self.weight, self.axis = original.weight, axis + + def forward(self, value): + weights = self.weight.chunk(size, dim=self.axis) + if self.axis == 0: + return Column.apply(value, self.weight) + partials = [F.linear(part.contiguous(), weight.contiguous()) + for part, weight in zip(value.chunk(size, dim=-1), weights, strict=True)] + result = partials[0] + for partial in partials[1:]: + result = result + partial + return result + + for layer in draft.layers: + for owner, names, axis in ( + (layer.self_attn, ("q_proj", "k_proj", "v_proj"), 0), + (layer.self_attn, ("o_proj",), 1), + (layer.mlp, ("gate_proj", "up_proj"), 0), + (layer.mlp, ("down_proj",), 1), + ): + for name in names: + setattr(owner, name, Projection(getattr(owner, name), axis)) + + +class ReferenceOptimizer: + def __init__(self, model, args): + self.parameters = [p for p in model.parameters() if p.requires_grad] + self.masters = [p.detach().float().clone().requires_grad_(True) for p in self.parameters] + self.gradients = [torch.zeros_like(master) for master in self.masters] + self.gradient_by_parameter = dict(zip(map(id, self.parameters), self.gradients, strict=True)) + self.optimizer = torch.optim.AdamW(self.masters, lr=args.learning_rate, + weight_decay=args.weight_decay, fused=True) + self.scheduler = LRSchedulerWithWarmup( + self.optimizer, max_lr=args.learning_rate, min_lr=args.min_lr, + total_steps=args.lr_total_steps, + warmup_steps=int(args.warmup_ratio * args.lr_total_steps), decay_style=args.lr_decay_style, + ) + + def accumulate(self): + # Match Core DDP's FP32 accumulation, independently of its hooks and + # buffers. Accumulating into BF16 leaf .grad changes the reference. + with torch.no_grad(): + for parameter, gradient in zip(self.parameters, self.gradients, strict=True): + gradient.add_(parameter.grad) + parameter.grad = None + + def step(self, max_norm, *, copy_model=True): + for gradient, master in zip(self.gradients, self.masters, strict=True): + master.grad = gradient + norm = torch.nn.utils.clip_grad_norm_(self.masters, max_norm) + if not torch.isfinite(norm).item(): + raise RuntimeError("Non-finite independent reference gradient") + self.optimizer.step() + self.scheduler.step() + if copy_model: + with torch.no_grad(): + torch._foreach_copy_(self.parameters, self.masters) + self.optimizer.zero_grad(set_to_none=True) + for parameter in self.parameters: + parameter.grad = None + torch._foreach_zero_(self.gradients) + return norm + + +def logical_value(tensor, parameter, mesh): + value = checkpoint_tensor(tensor.detach(), parameter, mesh) + return value.full_tensor() if isinstance(value, DTensor) else value + + +def vector_error(pairs): + sums = torch.zeros(2, dtype=torch.float64, device="cuda") + for value, expected in pairs: + delta = value.float() - expected.float() + sums[0] += delta.double().square().sum() + sums[1] += expected.double().square().sum() + return (sums[0] / sums[1].clamp_min(1e-30)).sqrt().item() + + +def cpu_snapshot(value): + if isinstance(value, torch.Tensor): + return value.detach().cpu().clone() + if isinstance(value, dict): + return {key: cpu_snapshot(item) for key, item in value.items()} + if isinstance(value, list): + return [cpu_snapshot(item) for item in value] + return copy.deepcopy(value) + + +def state_mismatches(current, expected, prefix=""): + if isinstance(expected, torch.Tensor): + return [] if torch.equal(current.detach().cpu(), expected) else [prefix] + if isinstance(expected, dict): + if set(current) != set(expected): + return [prefix + ".keys"] + return [name for key in expected for name in state_mismatches( + current[key], expected[key], f"{prefix}.{key}")] + if isinstance(expected, list): + if len(current) != len(expected): + return [prefix + ".length"] + return [name for index, item in enumerate(expected) for name in state_mismatches( + current[index], item, f"{prefix}.{index}")] + return [] if current == expected else [prefix] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--tp", type=int, default=2) + parser.add_argument("--sequence-length", type=int, default=128) + parser.add_argument("--anchors", type=int, default=16) + parser.add_argument("--attention-backend", choices=("fa4", "cudnn_flex_attention"), default="fa4") + parser.add_argument("--reference-tp-projections", action="store_true") + parser.add_argument("--gradient-checkpointing", action="store_true") + args = parser.parse_args() + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + dist.init_process_group("nccl") + init_gloo_group() + init_direct_cp_groups(1) + world, rank = dist.get_world_size(), dist.get_rank() + root = Path(__file__).resolve().parents[2] + os.environ.setdefault("ASPEC_TRAIN_DATA", str(root / "examples/data/sample_conversations.jsonl")) + os.environ.setdefault("ASPEC_EVAL_DATA", str(root / "examples/data/sample_conversations.jsonl")) + output = args.output.resolve() + if rank == 0: + output.mkdir(parents=True, exist_ok=False) + dist.barrier() + config = load_config(str(root / "configs/recipes/qwen3_8b_dspark.yaml"), cli_args=[ + f"model.target_model_path={args.model}", "model.target_model_backend=megatron", + f"dataset.train_data_path={root / 'examples/data/sample_conversations.jsonl'}", + f"dataset.eval_data_path={root / 'examples/data/sample_conversations.jsonl'}", + "training.train_backend=megatron", f"training.draft_tensor_parallel_size={args.tp}", + f"training.attention_backend={args.attention_backend}", + f"training.training_num_gpus_per_node={world}", "training.dflash_packing=false", + "training.draft_accumulation_steps=2", "training.deterministic_anchor_sampling=true", + f"training.gradient_checkpointing={str(args.gradient_checkpointing).lower()}", + f"training.max_seq_length={args.sequence_length}", f"training.dspark_num_anchors={args.anchors}", + "training.lr_total_steps=4", "training.num_train_steps=4", + "mooncake.destination_gpu_ring_slots=4", + "inference.inference_engine_type=megatron", "inference.target_prefill_memory_optimized=true", + "inference.inference_num_gpus=1", "inference.inference_num_gpus_per_engine=1", + "inference.megatron.tp_size=1", f"output_dir={output / 'run'}", + ]) + plan = compile_execution_plan(config) + runtime = plan.to_runtime_args() + runtime.rank, runtime.world_size = rank, world + draft_config = AutoDraftModelConfig.from_file(runtime.draft_model_config) + trainer = MegatronDSparkTrainer(runtime) + trainer.init_model(draft_config, args.model) + initial = model_checkpoint_state(trainer.draft_model, trainer.tp_mesh) + initial = {name: (value.full_tensor() if isinstance(value, DTensor) else value).cpu() + for name, value in initial.items()} + reference = reference_optimizer = update_oracle = None + if rank == 0: + with torch.device("cpu"): + reference_draft = DSparkDraftModel(copy.deepcopy(draft_config)).to(torch.bfloat16) + reference_draft.load_state_dict(initial, strict=True) + reference_draft.freeze_embedding() + if args.reference_tp_projections: + reference_tp_projections(reference_draft, args.tp) + reference = DSparkTrainer._build_training_wrapper(trainer, reference_draft.cuda()) + reference_optimizer = ReferenceOptimizer(reference_draft, runtime) + update_oracle = ReferenceOptimizer(reference_draft, runtime) + del initial + accumulation = runtime.draft_accumulation_steps + report = [] + + def train_window(step, compare_reference, *, capture_gradients=False): + native_num = native_den = None + reference_num = reference_den = None + reference_den_by_replica = {} + for micro in range(accumulation): + batch = fixture(draft_config, length=args.sequence_length, anchors=args.anchors, + step=step, micro=micro, replica=trainer.dp_rank) + with trainer.model.no_sync() if micro < accumulation - 1 else nullcontext(): + values = trainer.model(**batch, lm_head_weight=trainer.target_lm_head_weight) + num, den = values[-1] + (num / accumulation).backward() + native_num = num.detach().clone() if native_num is None else native_num + num.detach() + native_den = den.clone() if native_den is None else native_den + den + if compare_reference and rank == 0: + for replica in range(trainer.dp_size): + ref_batch = fixture(draft_config, length=args.sequence_length, anchors=args.anchors, + step=step, micro=micro, replica=replica) + ref_values = reference(**ref_batch, lm_head_weight=trainer.target_lm_head_weight) + ref_num, ref_den = ref_values[-1] + (ref_num / accumulation).backward() + reference_optimizer.accumulate() + reference_num = (ref_num.detach().clone() if reference_num is None + else reference_num + ref_num.detach()) + previous = reference_den_by_replica.get(replica) + reference_den_by_replica[replica] = ref_den.clone() if previous is None else previous + ref_den + denominator = trainer._normalize_accumulated_gradients(native_den, accumulation) + observed_gradients = (cpu_snapshot([p.main_grad for p in trainer.optimizer.model_params]) + if capture_gradients else None) + dist.all_reduce(native_num, group=trainer.dp_group) + if compare_reference and rank == 0: + reference_den = torch.stack(list(reference_den_by_replica.values())).sum(0) + with torch.no_grad(): + for gradient in reference_optimizer.gradients: + gradient.mul_(accumulation / reference_den) + gradient_pairs, before, expected_before = [], [], [] + if compare_reference and trainer.dp_rank == 0: + ref_parameters = dict(reference.draft_model.named_parameters()) if rank == 0 else {} + for name, parameter in trainer.draft_model.named_parameters(): + if not parameter.requires_grad: + continue + gradient = logical_value(parameter.main_grad, parameter, trainer.tp_mesh) + if rank == 0: + reference_parameter = ref_parameters[name] + gradient_pairs.append((gradient, reference_optimizer.gradient_by_parameter[ + id(reference_parameter)])) + update_oracle.gradient_by_parameter[id(reference_parameter)].copy_(gradient) + for index, parameter in enumerate(trainer.optimizer.model_params): + value = logical_value(trainer.optimizer.fp32_params[index], parameter, trainer.tp_mesh) + if rank == 0: + before.append(value.clone()) + if rank == 0: + expected_before = [p.detach().clone() for p in reference_optimizer.masters] + oracle_before = [p.detach().clone() for p in update_oracle.masters] + measurements = {} + if compare_reference and rank == 0: + measurements = { + "loss_relative_error": abs((native_num / denominator / (reference_num / reference_den) - 1).item()), + "gradient_relative_l2": vector_error(gradient_pairs), + "denominator": denominator.item(), "reference_denominator": reference_den.item(), + } + reference_optimizer.step(runtime.max_grad_norm) + oracle_norm = update_oracle.step(runtime.max_grad_norm, copy_model=False) + native_norm = trainer.optimizer.step() + if compare_reference and trainer.dp_rank == 0: + updates, same_gradient_updates = [], [] + for index, parameter in enumerate(trainer.optimizer.model_params): + value = logical_value(trainer.optimizer.fp32_params[index], parameter, trainer.tp_mesh) + if rank == 0: + updates.append((value - before[index], + reference_optimizer.masters[index] - expected_before[index])) + same_gradient_updates.append((value - before[index], + update_oracle.masters[index] - oracle_before[index])) + if rank == 0: + measurements["update_relative_l2"] = vector_error(updates) + measurements["same_gradient_adam_update_relative_l2"] = vector_error(same_gradient_updates) + measurements["clip_norm_relative_error"] = abs((native_norm / oracle_norm - 1).item()) + names = [name for name, parameter in trainer.draft_model.named_parameters() + if parameter.requires_grad] + contributions = [{"parameter": name, + "squared_error": (actual - expected).double().square().sum().item(), + "relative_l2": vector_error([(actual, expected)])} + for name, (actual, expected) in zip(names, updates, strict=True)] + measurements["largest_update_errors"] = sorted( + contributions, key=lambda item: item["squared_error"], reverse=True)[:5] + report.append(measurements) + dist.barrier() + return observed_gradients + + train_window(0, True) + checkpoint_root = output / "checkpoint" + state = {"model_state": ModelState(trainer.model), + "optim_state": OptimizerState(trainer.model, trainer.optimizer), + "lr_scheduler_state": LRSchedulerState(trainer.lr_scheduler)} + dcp.save(state, checkpoint_id=checkpoint_root) + saved_masters = cpu_snapshot(trainer.optimizer.fp32_params) + saved_model = cpu_snapshot({name: value for name, value in trainer.draft_model.state_dict().items() + if isinstance(value, torch.Tensor)}) + saved_optimizer = cpu_snapshot(trainer.optimizer.state_dict()) + saved_scheduler = cpu_snapshot(trainer.lr_scheduler.state_dict()) + rng = {"torch": torch.get_rng_state(), "cuda": torch.cuda.get_rng_state(), + "backend": trainer._checkpoint_backend_rng()} + expected_gradients = train_window(1, True, capture_gradients=True) + expected_masters = [p.detach().cpu().clone() for p in trainer.optimizer.fp32_params] + expected_optimizer = copy.deepcopy(trainer.optimizer.state_dict()) + expected_scheduler = copy.deepcopy(trainer.lr_scheduler.state_dict()) + # A same-state in-memory replay independently measures the arithmetic's + # determinism, before involving checkpoint deserialization. + trainer.draft_model.load_state_dict(saved_model, strict=True) + with torch.no_grad(): + for master, saved in zip(trainer.optimizer.fp32_params, saved_masters, strict=True): + master.copy_(saved) + trainer.optimizer.load_state_dict(saved_optimizer) + trainer.lr_scheduler.load_state_dict(saved_scheduler) + torch.set_rng_state(rng["torch"]) + torch.cuda.set_rng_state(rng["cuda"]) + trainer._restore_backend_rng(rng["backend"]) + memory_gradients = train_window(1, False, capture_gradients=True) + memory_exact = all(torch.equal(p.cpu(), expected) for p, expected in zip( + trainer.optimizer.fp32_params, expected_masters, strict=True)) + memory_gradient_error = vector_error(zip(memory_gradients, expected_gradients, strict=True)) + del memory_gradients + state["optim_state"] = OptimizerState(trainer.model, trainer.optimizer, load_path=checkpoint_root) + dcp.load(state, checkpoint_id=checkpoint_root) + load_errors = { + "masters": state_mismatches(trainer.optimizer.fp32_params, saved_masters), + "model": state_mismatches({name: value for name, value in trainer.draft_model.state_dict().items() + if isinstance(value, torch.Tensor)}, saved_model), + "optimizer": state_mismatches(trainer.optimizer.state_dict(), saved_optimizer), + "scheduler": state_mismatches(trainer.lr_scheduler.state_dict(), saved_scheduler), + } + print(json.dumps({"rank": rank, "checkpoint_load_errors": load_errors}), flush=True) + load_exact = torch.tensor(int(not any(load_errors.values())), device="cuda") + dist.all_reduce(load_exact, op=dist.ReduceOp.MIN) + del saved_model, saved_masters, saved_optimizer, saved_scheduler + torch.set_rng_state(rng["torch"]) + torch.cuda.set_rng_state(rng["cuda"]) + trainer._restore_backend_rng(rng["backend"]) + replay_gradients = train_window(1, False, capture_gradients=True) + replay_gradient_error = vector_error(zip(replay_gradients, expected_gradients, strict=True)) + del replay_gradients, expected_gradients + exact_masters = all(torch.equal(p.cpu(), expected) for p, expected in zip( + trainer.optimizer.fp32_params, expected_masters, strict=True)) + exact_optimizer = all(torch.equal(value, expected_optimizer["state"][index][field]) + for index, values in trainer.optimizer.state_dict()["state"].items() + for field, value in values.items()) + exact_scheduler = trainer.lr_scheduler.state_dict() == expected_scheduler + exact = torch.tensor(int(exact_masters and exact_optimizer and exact_scheduler), device="cuda") + dist.all_reduce(exact, op=dist.ReduceOp.MIN) + if rank == 0: + passed = exact.item() == 1 and load_exact.item() == 1 and all( + row["loss_relative_error"] <= 0.01 and row["gradient_relative_l2"] <= 0.03 + and row["update_relative_l2"] <= 0.03 + and row["denominator"] == row["reference_denominator"] for row in report) + result = {"passed": passed, "windows": report, "exact_checkpoint_replay": bool(exact.item()), + "exact_checkpoint_state": bool(load_exact.item()), + "local_load_errors": load_errors, + "local_replay_exact": {"masters": exact_masters, "optimizer": exact_optimizer, + "scheduler": exact_scheduler}, + "in_memory_replay_exact_masters": memory_exact, + "in_memory_replay_gradient_relative_l2": memory_gradient_error, + "checkpoint_replay_gradient_relative_l2": replay_gradient_error, + "tp": args.tp, "dp": trainer.dp_size, "sequence_length": args.sequence_length, + "anchors": args.anchors, "plan_id": plan.plan_id, + "reference_accumulation_dtype": "float32", + "reference_projection_topology": "simulated_tp" if args.reference_tp_projections else "unpartitioned", + "attention_backend": args.attention_backend, + "gradient_checkpointing": args.gradient_checkpointing, + "dspark_execution": trainer._kernel_observation(), + "tolerances": {"loss_relative_error": 0.01, "gradient_relative_l2": 0.03, + "update_relative_l2": 0.03}} + (output / "result.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result), flush=True) + else: + passed = None + verdict = [passed] + dist.broadcast_object_list(verdict, src=0) + dist.destroy_process_group() + if not verdict[0]: + raise SystemExit("DSpark Megatron execution alignment failed") + + +if __name__ == "__main__": + main() diff --git a/tools/validation/megatron_pipeline_acceptance.py b/tools/validation/megatron_pipeline_acceptance.py new file mode 100644 index 0000000..dd07ffa --- /dev/null +++ b/tools/validation/megatron_pipeline_acceptance.py @@ -0,0 +1,125 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Bounded mixed-backend train/resume/export through the normal ASpec launcher. + +Reuses ASpec's existing recipe audits. Unlike the bundled two-GPU Quickstart, +this validation entry keeps explicit TP, DP and gradient-accumulation settings. +Input capacity, future bitwise replay, quality and throughput are separate gates. +""" + +import argparse +import json +import os +import sys +from pathlib import Path + +from omegaconf import OmegaConf + +from aspec.commands.quickstart import build_profile +from aspec.commands.recipes import audit_run, digest, execute, export_asset + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument("--train-data", required=True, type=Path) + parser.add_argument("--eval-data", type=Path) + parser.add_argument("--target-python", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--target-backend", choices=("sglang", "megatron"), default="megatron") + parser.add_argument("--draft-backend", choices=("fsdp", "megatron"), default="megatron") + parser.add_argument("--target-tp", type=int, choices=(1, 2), default=1) + parser.add_argument("--draft-tp", type=int, choices=(1, 2), default=1) + parser.add_argument("--dp", type=int, default=1) + parser.add_argument("--accumulation", type=int, default=2) + parser.add_argument("--sequence-length", type=int, default=4096) + parser.add_argument("--anchors", type=int, default=512) + parser.add_argument("--steps", type=int, default=4) + parser.add_argument("--gradient-checkpointing", action="store_true") + parser.add_argument("--attention-backend", choices=("fa4", "cudnn_flex_attention"), + default="cudnn_flex_attention") + parser.add_argument("--target-address", default="http://127.0.0.1:30000") + args = parser.parse_args() + if min(args.dp, args.accumulation, args.sequence_length, args.anchors) < 1: + parser.error("DP, accumulation, sequence length and anchors must be positive") + if args.steps < 2 or args.steps % 2: + parser.error("Use an even step count >=2") + if args.target_backend == "sglang" and args.target_tp != 1: + parser.error("The retained SGLang Target path requires TP1") + if args.draft_backend == "fsdp" and args.draft_tp != 1: + parser.error("Draft TP requires the Megatron backend") + if not args.train_data.is_file() or not args.target_python.is_absolute() or not args.target_python.is_file(): + parser.error("Provide existing data and an absolute Target interpreter path") + if args.eval_data is not None and not args.eval_data.is_file(): + parser.error("Evaluation data must exist") + world = args.draft_tp * args.dp + budget = world + args.target_tp + devices = os.environ.get("CUDA_VISIBLE_DEVICES", "").split(",") + if len(devices) != budget or any(not item.strip() for item in devices) or len(set(devices)) != budget: + parser.error(f"Select exactly {budget} distinct Draft+Target GPUs") + root = args.output.resolve() + root.mkdir(parents=True, exist_ok=False) + config = build_profile("qwen3-8b", args.model) + config = OmegaConf.merge(config, OmegaConf.create({ + "dataset": {"train_data_path": str(args.train_data.resolve()), "chat_template": None, + "eval_data_path": str(args.eval_data.resolve()) if args.eval_data else None, + "eval_interval": args.steps // 2}, + "model": {"target_model_backend": args.target_backend}, + "training": { + "train_backend": args.draft_backend, "draft_tensor_parallel_size": args.draft_tp, + "training_num_gpus_per_node": world, "draft_accumulation_steps": args.accumulation, + "attention_backend": args.attention_backend, "dflash_packing": False, + "gradient_checkpointing": args.gradient_checkpointing, + "max_seq_length": args.sequence_length, "dspark_num_anchors": args.anchors, + "num_epochs": 100, "lr_total_steps": args.steps, + "save_interval": args.steps // 2, "save_final_checkpoint": True, + }, + "inference": { + "inference_engine_type": "megatron" if args.target_backend == "megatron" else "sgl", + "inference_num_gpus": args.target_tp, "inference_num_gpus_per_engine": args.target_tp, + "inference_num_gpus_per_node": budget, + "megatron": {"tp_size": args.target_tp}, + "target_service": {"mode": "managed-local", "address": args.target_address, + "python_executable": str(args.target_python)}, + }, + "mooncake": {"max_seq_len": args.sequence_length, + "destination_gpu_ring_slots": 3 if args.draft_backend == "megatron" else 1, + "source_gpu_ring_slots": max(12, 2 * args.dp)}, + })) + profile = root / "profile.yaml" + OmegaConf.save(config, profile) + result = {"status": "running", "scope": "functional train/resume/native export", + "source_input": {"path": str(args.train_data.resolve()), "sha256": digest(args.train_data)}, + "world_size": world, "dp": args.dp, "draft_tp": args.draft_tp, + "target_tp": args.target_tp, "accumulation": args.accumulation} + result["gradient_checkpointing"] = args.gradient_checkpointing + if args.eval_data: + result["eval_input"] = {"path": str(args.eval_data.resolve()), "sha256": digest(args.eval_data)} + try: + doctor = [sys.executable, "-m", "aspec.commands.doctor", "--gpu"] + if args.draft_backend == "megatron": + doctor += ["--backend", "megatron"] + execute(doctor, root / "doctor.log") + execute([sys.executable, "-m", "aspec.train_entry", "--config", profile, "--dry-run"], + root / "plan.log") + for label, limit in (("first", args.steps // 2), ("resumed", args.steps)): + run = root / label + overrides = [f"training.num_train_steps={limit}", f"cache_dir={root / 'cache'}"] + if label == "resumed": + overrides.append(f"training.load_path={root / 'first/checkpoints'}") + execute([sys.executable, "-m", "aspec.commands._recipe_smoke", "--config", profile, + "--output", run, "--gpus", budget, *overrides], root / f"{label}.log") + audit_run(run, limit, label == "resumed", require_eval=bool(args.eval_data), expected_ranks=world) + export_asset(root / "resumed/checkpoints" / f"iter_{args.steps + 1:07d}", root / "asset", + log_prefix=root / "asset") + result["status"] = "passed" + except BaseException as error: + result.update(status="failed", error=str(error)) + raise + finally: + (root / "acceptance.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result), flush=True) + + +if __name__ == "__main__": + main() diff --git a/tools/validation/megatron_target_alignment.py b/tools/validation/megatron_target_alignment.py new file mode 100644 index 0000000..19875c9 --- /dev/null +++ b/tools/validation/megatron_target_alignment.py @@ -0,0 +1,101 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Compare frozen upstream Qwen3 prefill with the unmodified HF model. + +Run with torchrun in the isolated Megatron environment. Captures use post-layer +residuals and the final normalized hidden state, matching ASpec's feature ABI. +This checks numerical correctness, not throughput or transfer lifecycle. +""" + +import argparse +import json +import os +from pathlib import Path + +import torch +import torch.distributed as dist + +from aspec.models.megatron import initialize_parallelism, load_qwen3_target + + +def error_metrics(value, reference): + value, reference = value.float(), reference.float() + delta = value - reference + return {"relative_l2": (delta.norm() / reference.norm().clamp_min(1e-12)).item(), + "max_absolute": delta.abs().max().item(), + "cosine": torch.nn.functional.cosine_similarity( + value.flatten(), reference.flatten(), dim=0).item()} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--sequence-length", type=int, default=128) + parser.add_argument("--implementation", choices=("local", "transformer_engine"), + default="transformer_engine") + args = parser.parse_args() + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + dist.init_process_group("nccl") + groups = initialize_parallelism(dist.get_world_size()) + model, config = load_qwen3_target(args.model, groups, implementation=args.implementation) + layers = [1, 9, 17, 25, 33] + captured = {} + for index in layers: + def capture(_module, _inputs, output, index=index): + value = output[0] if isinstance(output, tuple) else output + captured[str(index)] = value.squeeze(1).detach().clone() + + model.decoder.layers[index].register_forward_hook(capture) + + def finish(*, hidden_states, **_kwargs): + captured["final"] = hidden_states.squeeze(1).detach().clone() + + # Fixed tokens and positions on all TP ranks, independent of a tokenizer. + ids = (torch.arange(args.sequence_length, device="cuda") * 137 + 42) % config.vocab_size + versions = [p._version for p in model.parameters()] + with torch.inference_mode(): + model(input_ids=ids.unsqueeze(0), + position_ids=torch.arange(ids.numel(), device="cuda").unsqueeze(0), + attention_mask=None, output_processor=finish) + frozen = (not any(p.requires_grad for p in model.parameters()) + and versions == [p._version for p in model.parameters()]) + if dist.get_rank() == 0: + from transformers import AutoModelForCausalLM + + reference = AutoModelForCausalLM.from_pretrained( + args.model, dtype=torch.bfloat16, attn_implementation="sdpa", + trust_remote_code=False).cuda().eval().requires_grad_(False) + expected = {} + for index in layers: + def capture_reference(_module, _inputs, output, index=index): + value = output[0] if isinstance(output, tuple) else output + expected[str(index)] = value.squeeze(0).detach().clone() + + reference.model.layers[index].register_forward_hook(capture_reference) + with torch.inference_mode(): + expected["final"] = reference.model(input_ids=ids.unsqueeze(0)).last_hidden_state.squeeze(0) + measurements = {key: error_metrics(captured[key], value) for key, value in expected.items()} + # Declared before observing results; no per-layer fitted thresholds. + passed = frozen and all(m["relative_l2"] <= 0.02 and m["cosine"] >= 0.999 + for m in measurements.values()) + result = {"passed": passed, "target_frozen": frozen, "features": measurements, + "tensor_parallel": dist.get_world_size(), "sequence_length": ids.numel(), + "implementation": args.implementation, "torch": torch.__version__, + "cuda": torch.version.cuda, "device_capability": torch.cuda.get_device_capability(), + "module": str(Path(__import__("aspec").__file__).resolve()), + "tolerances": {"relative_l2_max": 0.02, "cosine_min": 0.999}} + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result), flush=True) + else: + passed = None + outcome = [passed] + dist.broadcast_object_list(outcome, src=0) + dist.destroy_process_group() + if not outcome[0]: + raise SystemExit("Qwen3 feature alignment failed; inspect the recorded measurements") + + +if __name__ == "__main__": + main()