Skip to content
Draft
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
265 changes: 254 additions & 11 deletions tests/cli/test_machines_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,29 @@
import requests


@pytest.fixture(autouse=True)
def _self_test_udp_probe_success(monkeypatch):
"""Default self-test machine tests to a successful external UDP echo probe."""
from vastai.cli.commands import machines

def _probe(public_ip, host_port, *, mapped_ports=None, attempts=3, timeout_seconds=2):
return True, {
"url": f"udp://{public_ip}:{host_port}",
"public_ip": public_ip,
"container_port": "5001/udp",
"external_port": str(host_port),
"host_port": str(host_port),
"timeout_seconds": timeout_seconds,
"attempt_count": 1,
"response_received": True,
"last_error_type": None,
"last_error": None,
"mapped_ports": sorted((mapped_ports or {}).keys()),
}

monkeypatch.setattr(machines, "probe_udp_echo", _probe)


class TestShowMachines:
def test_show_machines_raw(self, parse_argv, patch_get_client, mock_response):
patch_get_client.get.return_value = mock_response(200, {
Expand Down Expand Up @@ -107,7 +130,7 @@ def test_successful_destroy_does_not_warn_when_instance_is_already_gone(
"actual_status": "running",
"intended_status": "running",
"public_ipaddr": "127.0.0.1",
"ports": {"5000/tcp": [{"HostPort": "5000"}]},
"ports": {"5000/tcp": [{"HostPort": "5000"}], "5001/udp": [{"HostPort": "5001"}]},
"status_msg": "",
}

Expand Down Expand Up @@ -173,7 +196,7 @@ def test_ignore_requirements_warns_on_success(self, parse_argv, monkeypatch, cap
"intended_status": "running",
"actual_status": "running",
"public_ipaddr": "203.0.113.10",
"ports": {"5000/tcp": [{"HostPort": "5000"}]},
"ports": {"5000/tcp": [{"HostPort": "5000"}], "5001/udp": [{"HostPort": "5001"}]},
}

monkeypatch.setattr(machines.offers_api, "search_offers", Mock(return_value=[offer]))
Expand All @@ -193,7 +216,7 @@ def test_ignore_requirements_warns_on_success(self, parse_argv, monkeypatch, cap
assert "Requirement checks are skipped as a pass/fail gate" in out
assert "does not qualify this machine for verification" in out
assert out.count("does not qualify this machine for verification") >= 2
assert "Test passed." in out
assert "Machine ID 123 passed the self-test." in out

def test_ignore_requirements_warning_in_raw_summary(self, parse_argv, monkeypatch, capsys):
from vastai.cli.commands import machines
Expand Down Expand Up @@ -581,6 +604,28 @@ def test_preflight_direct_port_minimum_scales_by_gpu_count(self):
assert direct_ports["operator"] == ">="
assert "3 directly mapped ports per listed GPU" in direct_ports["purpose"]

def test_preflight_does_not_gate_on_virtual_cpu_count(self):
from vastai.cli.self_test.machine_diagnostics import (
failed_checks,
preflight_requirement_checks,
)

offer = _self_test_offer(
num_gpus=8,
gpu_ram=24 * 1024,
gpu_total_ram=8 * 24 * 1024,
cpu_ram=256 * 1024,
cpu_cores=1,
direct_port_count=24,
inet_down=600,
inet_up=600,
)

checks = preflight_requirement_checks(offer)

assert "cpu.cores" not in {check["id"] for check in checks}
assert failed_checks(checks) == []

def test_preflight_direct_port_overage_renders_advisory(
self, parse_argv, patch_get_client, monkeypatch, capsys
):
Expand Down Expand Up @@ -689,7 +734,7 @@ def test_ignore_requirements_runtime_success_preserves_preflight_as_metadata(
"actual_status": "running",
"intended_status": "running",
"public_ipaddr": "127.0.0.1",
"ports": {"5000/tcp": [{"HostPort": "5000"}]},
"ports": {"5000/tcp": [{"HostPort": "5000"}], "5001/udp": [{"HostPort": "5001"}]},
"status_msg": "",
}
monkeypatch.setattr(
Expand Down Expand Up @@ -774,6 +819,7 @@ def test_default_cuda_mapping_still_selects_official_image(
assert create.call_args.kwargs["runtype"] == "ssh_direc ssh_proxy"
assert create.call_args.kwargs["label"] == "vast-self-test-machine-42"
assert result["diagnostics"]["launch"]["label"] == "vast-self-test-machine-42"
assert result["diagnostics"]["launch"]["ports"] == ["5000/tcp", "1234/tcp", "5001/udp"]

def test_cuda_mapping_selects_cuda_133_exact_match(
self, parse_argv, patch_get_client, monkeypatch
Expand Down Expand Up @@ -996,7 +1042,7 @@ def test_missing_progress_port_reports_available_ports(
assert result["diagnostics"]["runtime_failure"]["progress_endpoint"] == endpoint
assert destroy.call_count >= 1

def test_progress_endpoint_never_reachable_records_endpoint_diagnostic(
def test_missing_udp_port_reports_available_ports(
self, parse_argv, patch_get_client, monkeypatch
):
offer = _self_test_offer()
Expand All @@ -1023,6 +1069,203 @@ def test_progress_endpoint_never_reachable_records_endpoint_diagnostic(
destroy = Mock(return_value={"success": True})
monkeypatch.setattr("vastai.cli.commands.machines.instances_api.destroy_instance", destroy)
monkeypatch.setattr("vastai.cli.commands.machines.time.sleep", lambda *_: None)

args = parse_argv(["self-test", "machine", "42", "--raw"])
result = args.func(args)

udp_probe = result["diagnostics"]["udp_probe"]
assert result["failure_code"] == "udp_port_not_mapped"
assert udp_probe["container_port"] == "5001/udp"
assert udp_probe["external_port"] is None
assert udp_probe["mapped_ports"] == ["22/tcp", "5000/tcp"]
assert result["diagnostics"]["runtime_failure"]["udp_probe"] == udp_probe
assert destroy.call_count >= 1

def test_udp_probe_failure_after_tcp_success_is_distinct(
self, parse_argv, patch_get_client, monkeypatch, capsys
):
offer = _self_test_offer()
running_instance = {
"id": 123,
"actual_status": "running",
"intended_status": "running",
"public_ipaddr": "127.0.0.1",
"ports": {"5000/tcp": [{"HostPort": "45000"}], "5001/udp": [{"HostPort": "45001"}]},
"status_msg": "",
}
udp_diagnostic = {
"url": "udp://127.0.0.1:45001",
"public_ip": "127.0.0.1",
"container_port": "5001/udp",
"external_port": "45001",
"host_port": "45001",
"timeout_seconds": 2,
"attempt_count": 3,
"response_received": False,
"last_error_type": "TimeoutError",
"last_error": "timed out",
"mapped_ports": ["5000/tcp", "5001/udp"],
}
monkeypatch.setattr(
"vastai.cli.commands.machines.offers_api.search_offers",
Mock(return_value=[offer]),
)
monkeypatch.setattr(
"vastai.cli.commands.machines.instances_api.create_instance",
Mock(return_value={"new_contract": 123}),
)
monkeypatch.setattr(
"vastai.cli.commands.machines.instances_api.show_instance",
Mock(return_value=running_instance),
)
destroy = Mock(return_value={"success": True})
monkeypatch.setattr("vastai.cli.commands.machines.instances_api.destroy_instance", destroy)
monkeypatch.setattr("vastai.cli.commands.machines.time.sleep", lambda *_: None)
monkeypatch.setattr(
"vastai.cli.commands.machines.requests.get",
Mock(return_value=SimpleNamespace(status_code=200, text="Starting tests...")),
)
monkeypatch.setattr(
"vastai.cli.commands.machines.probe_udp_echo",
Mock(return_value=(False, udp_diagnostic)),
)

args = parse_argv(["self-test", "machine", "42"])
with pytest.raises(SystemExit) as exc_info:
args.func(args)

captured = capsys.readouterr()
assert exc_info.value.code == 1
assert "Successfully established HTTPS connection to the server." in captured.out
assert "UDP self-test probe failed after TCP progress endpoint was reachable." in captured.out
assert "External UDP port tested: 45001" in captured.out
assert "- code: udp_probe_failed" in captured.out
assert "- UDP tried: udp://127.0.0.1:45001" in captured.out
assert destroy.call_count >= 1

def test_wait_for_instance_loading_status_is_compact_without_debugging(
self, parse_argv, patch_get_client, monkeypatch, capsys
):
offer = _self_test_offer()
loading_instance = {
"id": 123,
"actual_status": "loading",
"intended_status": "running",
"status_msg": "ff81e2caff08: Verifying Checksum\nff81e2caff08: Download complete",
}
running_instance = {
"id": 123,
"actual_status": "running",
"intended_status": "running",
"public_ipaddr": "127.0.0.1",
"ports": {"5000/tcp": [{"HostPort": "45000"}], "5001/udp": [{"HostPort": "45001"}]},
"status_msg": "",
}
monkeypatch.setattr(
"vastai.cli.commands.machines.offers_api.search_offers",
Mock(return_value=[offer]),
)
monkeypatch.setattr(
"vastai.cli.commands.machines.instances_api.create_instance",
Mock(return_value={"new_contract": 123}),
)
monkeypatch.setattr(
"vastai.cli.commands.machines.instances_api.show_instance",
Mock(side_effect=[loading_instance, loading_instance, running_instance, running_instance]),
)
destroy = Mock(return_value={"success": True})
monkeypatch.setattr("vastai.cli.commands.machines.instances_api.destroy_instance", destroy)
monkeypatch.setattr("vastai.cli.commands.machines.requests.get", Mock(return_value=SimpleNamespace(status_code=200, text="DONE")))
monkeypatch.setattr("vastai.cli.commands.machines.time.sleep", lambda *_: None)

args = parse_argv(["self-test", "machine", "42"])
with pytest.raises(SystemExit) as exc_info:
args.func(args)

captured = capsys.readouterr()
assert exc_info.value.code == 0
assert "Instance 123 is loading; waiting for running status..." in captured.out
assert "Still loading... 0s elapsed" in captured.out
assert "Instance 123 is ready after 0s." in captured.out
assert "status: loading" not in captured.out
assert "status_msg:" not in captured.out
assert "Verifying Checksum" not in captured.out
assert destroy.call_count >= 1

def test_wait_for_instance_loading_status_is_verbose_with_debugging(
self, parse_argv, patch_get_client, monkeypatch, capsys
):
offer = _self_test_offer()
loading_instance = {
"id": 123,
"actual_status": "loading",
"intended_status": "running",
"status_msg": "ff81e2caff08: Verifying Checksum\nff81e2caff08: Download complete",
}
running_instance = {
"id": 123,
"actual_status": "running",
"intended_status": "running",
"public_ipaddr": "127.0.0.1",
"ports": {"5000/tcp": [{"HostPort": "45000"}], "5001/udp": [{"HostPort": "45001"}]},
"status_msg": "",
}
monkeypatch.setattr(
"vastai.cli.commands.machines.offers_api.search_offers",
Mock(return_value=[offer]),
)
monkeypatch.setattr(
"vastai.cli.commands.machines.instances_api.create_instance",
Mock(return_value={"new_contract": 123}),
)
monkeypatch.setattr(
"vastai.cli.commands.machines.instances_api.show_instance",
Mock(side_effect=[loading_instance, running_instance, running_instance]),
)
destroy = Mock(return_value={"success": True})
monkeypatch.setattr("vastai.cli.commands.machines.instances_api.destroy_instance", destroy)
monkeypatch.setattr("vastai.cli.commands.machines.requests.get", Mock(return_value=SimpleNamespace(status_code=200, text="DONE")))
monkeypatch.setattr("vastai.cli.commands.machines.time.sleep", lambda *_: None)

args = parse_argv(["self-test", "machine", "42", "--debugging"])
with pytest.raises(SystemExit) as exc_info:
args.func(args)

captured = capsys.readouterr()
assert exc_info.value.code == 0
assert "Instance 123 status: loading / intended: running; waiting for 'running' status." in captured.out
assert "status_msg: ff81e2caff08: Verifying Checksum" in captured.out
assert "Instance 123 is loading; waiting for running status" not in captured.out
assert "Still loading..." not in captured.out
assert destroy.call_count >= 1

def test_progress_endpoint_never_reachable_records_endpoint_diagnostic(
self, parse_argv, patch_get_client, monkeypatch
):
offer = _self_test_offer()
running_instance = {
"id": 123,
"actual_status": "running",
"intended_status": "running",
"public_ipaddr": "127.0.0.1",
"ports": {"5000/tcp": [{"HostPort": "45000"}], "22/tcp": [{"HostPort": "40022"}], "5001/udp": [{"HostPort": "45001"}]},
"status_msg": "",
}
monkeypatch.setattr(
"vastai.cli.commands.machines.offers_api.search_offers",
Mock(return_value=[offer]),
)
monkeypatch.setattr(
"vastai.cli.commands.machines.instances_api.create_instance",
Mock(return_value={"new_contract": 123}),
)
monkeypatch.setattr(
"vastai.cli.commands.machines.instances_api.show_instance",
Mock(return_value=running_instance),
)
destroy = Mock(return_value={"success": True})
monkeypatch.setattr("vastai.cli.commands.machines.instances_api.destroy_instance", destroy)
monkeypatch.setattr("vastai.cli.commands.machines.time.sleep", lambda *_: None)
monkeypatch.setattr(
"vastai.cli.commands.machines.requests.get",
Mock(side_effect=requests.exceptions.ConnectTimeout("timed out for ?api_key=secret")),
Expand All @@ -1042,7 +1285,7 @@ def test_progress_endpoint_never_reachable_records_endpoint_diagnostic(
assert endpoint["last_error_type"] == "ConnectTimeout"
assert "api_key=secret" not in endpoint["last_error"]
assert "api_key=REDACTED" in endpoint["last_error"]
assert endpoint["mapped_ports"] == ["22/tcp", "5000/tcp"]
assert endpoint["mapped_ports"] == ["22/tcp", "5000/tcp", "5001/udp"]
assert result["diagnostics"]["runtime_failure"]["progress_endpoint"] == endpoint
assert destroy.call_count >= 1

Expand All @@ -1055,7 +1298,7 @@ def test_progress_endpoint_failure_prints_external_port(
"actual_status": "running",
"intended_status": "running",
"public_ipaddr": "127.0.0.1",
"ports": {"5000/tcp": [{"HostPort": "45000"}], "22/tcp": [{"HostPort": "40022"}]},
"ports": {"5000/tcp": [{"HostPort": "45000"}], "22/tcp": [{"HostPort": "40022"}], "5001/udp": [{"HostPort": "45001"}]},
"status_msg": "",
}
monkeypatch.setattr(
Expand Down Expand Up @@ -1088,7 +1331,7 @@ def test_progress_endpoint_failure_prints_external_port(
assert exc_info.value.code == 1
assert "External port tested: 45000" in captured.out
assert "- external port tested: 45000" in captured.out
assert "- mapped container ports: 22/tcp, 5000/tcp" in captured.out
assert "- mapped container ports: 22/tcp, 5000/tcp, 5001/udp" in captured.out

def test_progress_endpoint_lost_after_success_records_different_failure(
self, parse_argv, patch_get_client, monkeypatch
Expand All @@ -1099,7 +1342,7 @@ def test_progress_endpoint_lost_after_success_records_different_failure(
"actual_status": "running",
"intended_status": "running",
"public_ipaddr": "127.0.0.1",
"ports": {"5000/tcp": [{"HostPort": "45000"}]},
"ports": {"5000/tcp": [{"HostPort": "45000"}], "5001/udp": [{"HostPort": "45001"}]},
"status_msg": "",
}
monkeypatch.setattr(
Expand Down Expand Up @@ -1150,7 +1393,7 @@ def test_progress_endpoint_http_non_200_records_status_code(
"actual_status": "running",
"intended_status": "running",
"public_ipaddr": "127.0.0.1",
"ports": {"5000/tcp": [{"HostPort": "45000"}]},
"ports": {"5000/tcp": [{"HostPort": "45000"}], "5001/udp": [{"HostPort": "45001"}]},
"status_msg": "",
}
monkeypatch.setattr(
Expand Down Expand Up @@ -1193,7 +1436,7 @@ def test_progress_endpoint_empty_200_records_empty_timeout(
"actual_status": "running",
"intended_status": "running",
"public_ipaddr": "127.0.0.1",
"ports": {"5000/tcp": [{"HostPort": "45000"}]},
"ports": {"5000/tcp": [{"HostPort": "45000"}], "5001/udp": [{"HostPort": "45001"}]},
"status_msg": "",
}
monkeypatch.setattr(
Expand Down
2 changes: 1 addition & 1 deletion tests/cli/test_self_test_support_bundle.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ def test_self_test_bundle_creation_error_preserves_original_failure(
captured = capsys.readouterr()
assert exc_info.value.code == 1
assert "WARNING: failed to create self-test diagnostic bundle: disk full" in captured.out
assert "Test failed: 8 preflight requirement check(s) failed." in captured.out
assert "Test failed: 7 preflight requirement check(s) failed." in captured.out


def test_self_test_runtime_failure_bundle_includes_instance_logs(
Expand Down
Loading
Loading