Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ dist
.env.*
**/ray-auth-token
remote_exec.sh
AGENTS.override.md
**/*.safetensors
**/*.pt
container-cache
Expand Down
11 changes: 10 additions & 1 deletion aspec/_environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,15 @@ def command_environment():
raise ValueError("ASPEC_CUDA_COMPAT_DIR must contain libcuda.so.1.")
paths.append(str(path))
purelib = Path(sysconfig.get_paths()["purelib"])
cudnn = purelib / "nvidia/cudnn"
if not env.get("CUDNN_HOME") and (cudnn / "lib/libcudnn.so.9").is_file():
# TE searches CUDA_HOME and the host toolkit before LD_LIBRARY_PATH.
# Bind its explicit search to this interpreter's cuDNN wheel.
env["CUDNN_HOME"] = str(cudnn)
env["ASPEC_AUTO_CUDNN_HOME"] = str(cudnn)
toolkit = purelib / "nvidia/cu13"
if (toolkit / "lib/libcudart.so.13").is_file():
env.setdefault("CUDNN_FRONTEND_CUDART_LIB_NAME", "libcudart.so.13")
if not env.get("CUDA_HOME") and not env.get("CUDA_PATH") and (toolkit / "bin/nvcc").is_file():
view = _toolkit_view(toolkit, env)
env["CUDA_HOME"] = str(view)
Expand All @@ -101,7 +109,8 @@ def command_environment():
previous = env.get("LIBRARY_PATH", "")
env["LIBRARY_PATH"] = str(view / "lib64") + (os.pathsep + previous if previous else "")
paths.append(str(view / "lib64"))
paths.extend(str(p) for p in (purelib / "nvidia/cuda_runtime/lib", purelib / "torch/lib")
paths.extend(str(p) for p in (purelib / "nvidia/cuda_runtime/lib", purelib / "torch/lib",
purelib / "nvidia/cudnn/lib", purelib / "nvidia/nccl/lib")
if p.is_dir())
if (toolkit / "lib").is_dir():
paths.append(str(toolkit / "lib"))
Expand Down
3 changes: 3 additions & 0 deletions aspec/_target_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ def main():
automatic_toolkit = os.environ.pop("ASPEC_AUTO_CUDA_HOME", None)
if automatic_toolkit and os.environ.get("CUDA_HOME") == automatic_toolkit:
os.environ.pop("CUDA_HOME", None)
automatic_cudnn = os.environ.pop("ASPEC_AUTO_CUDNN_HOME", None)
if automatic_cudnn and os.environ.get("CUDNN_HOME") == automatic_cudnn:
os.environ.pop("CUDNN_HOME", None)
os.environ["PATH"] = str(Path(sys.executable).parent) + os.pathsep + os.environ.get("PATH", "")
if cache := os.environ.get("ASPEC_CACHE_HOME"):
if Path(cache).name != "target":
Expand Down
5 changes: 4 additions & 1 deletion aspec/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,10 @@ def main():
else:
module = "aspec.train_entry"
help_requested = arguments in (["--help"], ["-h"])
if module == "aspec.inference.service" and not help_requested and not is_target_environment():
megatron_target = ("--backend=megatron" in arguments or any(
a == "--backend" and b == "megatron" for a, b in zip(arguments, arguments[1:])
))
if module == "aspec.inference.service" and not help_requested and not is_target_environment() and not megatron_target:
raise SystemExit("Start aspec target in the Target environment (pip install ling-aspec-target)")
try:
execute([sys.executable, "-m", module, *arguments])
Expand Down
31 changes: 23 additions & 8 deletions aspec/commands/_checkpoint_audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,19 +21,19 @@
import math


def cpu_values(value):
def cpu_values(value, *, full_tensors=False):
import torch

if isinstance(value, torch.Tensor):
if hasattr(value, "to_local"):
value = value.to_local()
value = value.full_tensor() if full_tensors else value.to_local()
return value.detach().cpu().clone()
if isinstance(value, dict):
return {key: cpu_values(item) for key, item in value.items()}
return {key: cpu_values(item, full_tensors=full_tensors) for key, item in value.items()}
if isinstance(value, list):
return [cpu_values(item) for item in value]
return [cpu_values(item, full_tensors=full_tensors) for item in value]
if isinstance(value, tuple):
return tuple(cpu_values(item) for item in value)
return tuple(cpu_values(item, full_tensors=full_tensors) for item in value)
return value


Expand Down Expand Up @@ -80,8 +80,12 @@ def compare_checkpoint(trainer, directory):

candidate_model = ModelState(trainer.model).state_dict()
candidate_optimizer = OptimizerState(trainer.model, trainer.optimizer).state_dict()
original_model = cpu_values(candidate_model)
original_optimizer = cpu_values(candidate_optimizer)
# Core compute parameters are ordinary local tensors, while its DCP state
# describes logical TP tensors. Read full independent CPU references rather
# than treating a local shard as the checkpoint's complete destination.
logical_tp = getattr(trainer, "execution_backend", "fsdp") == "megatron"
original_model = cpu_values(candidate_model, full_tensors=logical_tp)
original_optimizer = cpu_values(candidate_optimizer, full_tensors=logical_tp)
dcp.load({"model_state": original_model}, checkpoint_id=str(directory / "model"))
dcp.load({"optim_state": original_optimizer}, checkpoint_id=str(directory / "optimizer"))
statistics = {
Expand All @@ -91,6 +95,8 @@ def compare_checkpoint(trainer, directory):
worst_parameters = {key: {"relative_l2": 0, "name": None} for key in statistics}

def compare(key, name, reference, actual):
if logical_tp and hasattr(actual, "full_tensor"):
actual = actual.full_tensor()
reference = reference.reshape(-1)
actual = actual.reshape(-1)
if reference.shape != actual.shape:
Expand Down Expand Up @@ -127,16 +133,25 @@ def compare(key, name, reference, actual):
}


def backend_rng_identity(state):
return {"backend": state["backend"],
"cuda_tracker": tensor_hash(sorted(state["cuda_tracker"].items()))}


def audit_state(trainer):
import torch

return {
result = {
"global_step": trainer.global_step,
"state": state_hashes(trainer),
"scheduler": cpu_values(trainer.lr_scheduler.state_dict()),
"torch_rng_sha256": hashlib.sha256(torch.random.get_rng_state().numpy().tobytes()).hexdigest(),
"cuda_rng_sha256": hashlib.sha256(torch.cuda.get_rng_state().cpu().numpy().tobytes()).hexdigest(),
}
snapshot = getattr(trainer, "_checkpoint_backend_rng", None)
if snapshot is not None:
result["backend_rng"] = backend_rng_identity(snapshot())
return result


def add_statistics(total, reference, actual):
Expand Down
5 changes: 5 additions & 0 deletions aspec/commands/_recipe_smoke.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,11 @@ def init(self, args, role, mooncake_config=None, with_ref=False):
}
if any(initial[key] != value for key, value in hashes.items()):
raise AssertionError("Restored RNG differs from saved checkpoint")
if "backend_rng" in initial:
from aspec.commands._checkpoint_audit import backend_rng_identity

if backend_rng_identity(rng["backend_rng"]) != initial["backend_rng"]:
raise AssertionError("Restored model-parallel RNG differs from saved checkpoint")
(self._audit_root / f"recipe-resume-diff{self._audit_suffix}.json").write_text(
json.dumps({"state": compared, "rng_exact": True}, sort_keys=True)
)
Expand Down
72 changes: 68 additions & 4 deletions aspec/commands/doctor.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,16 +53,79 @@ def check_dependency_metadata(distributions=None):
return result


def expected_versions(target=False):
source = asset_path("requirements/sglang-v0.5.21.txt" if target else "requirements/draft.txt")
def expected_versions(target=False, *, backend=None):
manifest = ("megatron-v0.20.0.txt" if backend == "megatron" else
"sglang-v0.5.21.txt" if target else "draft.txt")
source = asset_path("requirements/" + manifest)
requirements = [Requirement(line) for line in source.read_text().splitlines()
if line.strip() and not line.startswith("#")]
return {requirement.name: next((spec.version for spec in requirement.specifier
if spec.operator == "=="), None)
for requirement in requirements}


def inspect_environment(*, gpu=False, target=False):
def _inspect_megatron_environment(*, gpu, target):
result = {"python": sys.executable, "prefix": sys.prefix, "platform": platform.platform(),
"gpu_requested": gpu, "target_requested": target, "backend": "megatron",
"role": "target" if target else "draft", "patch": "not_required",
"checks": [], "errors": [], "dependencies": check_dependency_metadata()}
result["errors"].extend(result["dependencies"]["conflicts"])
if error := venv_error():
result["errors"].append(error)
if sys.version_info[:2] < (3, 12) or sys.platform != "linux":
result["errors"].append("Megatron requires Linux with Python >=3.12")
for name, expected in expected_versions(backend="megatron").items():
try:
actual = importlib.metadata.version(name)
except importlib.metadata.PackageNotFoundError:
actual = None
ok = actual is not None and (expected is None or actual.split("+", 1)[0] == expected)
result["checks"].append({"package": name, "expected": expected, "installed": actual, "ok": ok})
if not ok:
result["errors"].append(f"{name}: expected {expected}, found {actual}")
for name in ("ling-aspec", "ling-aspec-target", "sglang"):
try:
importlib.metadata.version(name)
except importlib.metadata.PackageNotFoundError:
pass
else:
result["errors"].append(f"{name} belongs in a separate backend environment")
if gpu:
try:
import torch
import transformer_engine.pytorch as te
from mooncake.engine import TransferEngine

from aspec.models.megatron import require_megatron

require_megatron()
layer = te.Linear(16, 16, params_dtype=torch.bfloat16, device="cuda")
output = layer(torch.ones(2, 16, dtype=torch.bfloat16, device="cuda"))
if not torch.isfinite(output).all().item():
raise RuntimeError("Transformer Engine BF16 linear produced non-finite values")
result["cuda"] = {"torch": torch.__version__, "runtime": torch.version.cuda,
"device_capability": list(torch.cuda.get_device_capability()),
"transformer_engine": te.__file__,
"mooncake_native_import": TransferEngine is not None}
except Exception as error:
result["errors"].append(f"Megatron GPU/native-library check failed: {error}")
result["ok"] = not result["errors"]
return result


def inspect_environment(*, gpu=False, target=False, backend=None):
if backend is None:
marker = Path(sys.prefix) / ".aspec-backend"
if marker.is_file() and marker.read_text().strip() == "megatron":
backend = "megatron"
else:
try:
importlib.metadata.version("ling-aspec-megatron")
backend = "megatron"
except importlib.metadata.PackageNotFoundError:
pass
if backend == "megatron":
return _inspect_megatron_environment(gpu=gpu, target=target)
result = {"python": sys.executable, "prefix": sys.prefix, "platform": platform.platform(),
"gpu_requested": gpu, "target_requested": target, "checks": [], "errors": []}
result["dependencies"] = check_dependency_metadata()
Expand Down Expand Up @@ -137,9 +200,10 @@ def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--gpu", action="store_true", help="Also allocate a small CUDA tensor and import Mooncake")
parser.add_argument("--target", action="store_true", help="Also require SGLang and its training patch")
parser.add_argument("--backend", choices=("sglang", "megatron"), default=None)
parser.add_argument("--json", action="store_true")
args = parser.parse_args()
result = inspect_environment(gpu=args.gpu, target=args.target)
result = inspect_environment(gpu=args.gpu, target=args.target, backend=args.backend)
if args.json:
print(json.dumps(result, indent=2))
else:
Expand Down
10 changes: 10 additions & 0 deletions aspec/config/inference_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,15 @@
from aspec.config.mooncake_config import MooncakeConfig


@dataclass
class MegatronConfig:
"""Node-local, frozen Qwen3 prefill through upstream Megatron Core."""

tp_size: int = 1
init_timeout: int = 600
implementation: str = "transformer_engine"


@dataclass
class SGLangConfig:
"""Essential SGLang engine configuration.
Expand Down Expand Up @@ -195,6 +204,7 @@ class InferenceConfig:
# performance default; the Plan must explicitly admit or omit it.
target_prefill_memory_optimized: bool = False
offline: OfflineTrainingConfig = field(default_factory=OfflineTrainingConfig)
megatron: MegatronConfig = field(default_factory=MegatronConfig)
sglang: SGLangConfig = field(default_factory=SGLangConfig)
vllm: VllmConfig = field(default_factory=VllmConfig)
trtllm: TrtllmConfig = field(default_factory=TrtllmConfig)
Expand Down
82 changes: 82 additions & 0 deletions aspec/config/megatron.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
# Copyright 2026 Ant Group Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
"""Dependency-light capability boundary for the upstream Megatron integration."""

MEGATRON_VERSION = "0.20.0"
MEGATRON_REVISION = "685fb8bd970185e9ac3283fe32a5d6667bf4b268"


def validate_megatron_execution(args, draft):
"""Reject combinations outside the runnable dense DSpark CP1/PP1 lane."""
training = getattr(args, "train_backend", "fsdp")
target = getattr(args, "target_model_backend", "sglang")
tp = getattr(args, "draft_tensor_parallel_size", 1)
if type(tp) is not int or tp < 1:
raise ValueError("training.draft_tensor_parallel_size must be a positive integer")
if training != "megatron" and tp != 1:
raise ValueError("Draft TP requires training.train_backend=megatron")
if training != "megatron" and target != "megatron":
return
if getattr(args, "cluster_mode", "single_node") != "single_node":
raise ValueError("Megatron currently requires single-node execution")
if getattr(args, "aspec_shared_source", False) or getattr(args, "aspec_shared_reader", None):
raise ValueError("Megatron shared Target execution is not supported yet")
if draft["family"] != "dspark":
raise ValueError("Megatron currently supports the DSpark objective")
if draft.get("mtp_conditioning"):
raise ValueError("Megatron currently requires an unconditioned DSpark Draft")
if getattr(args, "draft_context_parallel_size", 1) != 1:
raise ValueError("Megatron currently requires Draft CP1")
if target == "megatron":
if getattr(args, "sglang_extra_args", None) or getattr(args, "sglang_allow_context_length_override", False):
raise ValueError("Megatron Target does not apply SGLang backend overrides")
if getattr(args, "megatron_implementation", "transformer_engine") not in {"local", "transformer_engine"}:
raise ValueError("Unknown Megatron model implementation")
if not getattr(args, "target_prefill_memory_optimized", False):
raise ValueError("Megatron Target requires target_prefill_memory_optimized=true")
if getattr(args, "sglang_nnodes", 1) != 1 or getattr(args, "sglang_pp_size", 1) != 1:
raise ValueError("Megatron Target currently requires node-local TP and PP1")
if getattr(args, "train_with_decode", False):
raise ValueError("Megatron Target supports prefill-only execution")
target_tp = getattr(args, "megatron_tp_size", 1)
if type(target_tp) is not int or target_tp < 1:
raise ValueError("inference.megatron.tp_size must be a positive integer")
if target_tp not in {1, 2}:
raise ValueError("Megatron Target currently validates TP1/TP2")
if target_tp != getattr(args, "inference_num_gpus_per_engine", 1):
raise ValueError("inference.megatron.tp_size must equal inference_num_gpus_per_engine")
slots = (getattr(args, "mooncake_source_gpu_ring_slots", None)
or getattr(args, "mooncake_gpu_ring_slots", 2))
if getattr(args, "inference_batch_size", 1) > slots:
raise ValueError("Megatron Target source ring must retain one complete inference batch")
if training == "megatron":
if getattr(args, "distributed_backend", "nccl") != "nccl":
raise ValueError("Megatron Draft requires the NCCL distributed backend")
if getattr(args, "micro_batch_size", 1) != 1:
raise ValueError("Megatron Draft currently requires micro_batch_size=1")
prefetch = max(int(getattr(args, "prefetch_depth", 0)), 0)
destination_slots = (getattr(args, "mooncake_destination_gpu_ring_slots", None)
or getattr(args, "mooncake_gpu_ring_slots", 2))
if prefetch and destination_slots < prefetch + 2:
raise ValueError("Megatron source prefetch requires destination_gpu_ring_slots >= prefetch_depth + 2")
if tp not in {1, 2}:
raise ValueError("Megatron Draft currently validates TP1/TP2")
if getattr(args, "inference_engine_type", "sgl") == "offline":
raise ValueError("Offline replay with Megatron Draft is not validated")
if getattr(args, "fsdp_strategy", "REPLICATE") != "REPLICATE":
raise ValueError("Megatron Draft currently uses replicated DDP rather than FSDP sharding")
if getattr(args, "fsdp_cpu_offload", False):
raise ValueError("Megatron Draft CPU offload is not supported")
if getattr(args, "world_size", 1) % tp:
raise ValueError("Draft world size must be divisible by draft_tensor_parallel_size")
if getattr(args, "dflash_packing", False):
raise ValueError("Megatron Draft packed execution is not supported yet")
if getattr(args, "compile_model", False):
raise ValueError("Megatron Draft currently compiles individual optimized kernels only")
if draft.get("structure", {}).get("ffn_type", "dense") != "dense":
raise ValueError("Megatron Draft currently requires a dense FFN")
attention = draft.get("attention", {})
if (attention.get("head_layout", {}).get("type", "gqa") not in {"gqa", "mha"}
or any(window is not None for window in attention.get("context_windows", []))
or any(attention.get(kind) is not None for kind in ("csa", "qsa", "dsa"))):
raise ValueError("Megatron Draft currently requires dense full GQA/MHA attention")
Loading
Loading