Skip to content

Commit ad31a42

Browse files
AWarnoAWarno
committed
Merge branch 'awarno/haproxy' of https://github.com/NVIDIA-NeMo/Evaluator into awarno/haproxy
2 parents b0afe69 + a1df099 commit ad31a42

File tree

2 files changed

+5
-59
lines changed

2 files changed

+5
-59
lines changed

‎packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/package_info.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
# Below is the _next_ version that will be published, not the currently published one.
1717
MAJOR = 0
1818
MINOR = 1
19-
PATCH = 20
19+
PATCH = 21
2020
PRE_RELEASE = ""
2121

2222
# Use the following formatting: (major, minor, patch, pre-release)

‎packages/nemo-evaluator-launcher/tests/unit_tests/test_slurm_executor.py‎

Lines changed: 4 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@ def base_config(self):
4545
"image": "test-image:latest",
4646
"command": "test-command",
4747
"served_model_name": "test-model",
48+
"port": 8000,
49+
"endpoints": {
50+
"health": "/health",
51+
},
4852
},
4953
"execution": {
5054
"type": "slurm",
@@ -83,12 +87,6 @@ def mock_dependencies(self):
8387
patch(
8488
"nemo_evaluator_launcher.executors.slurm.executor.get_task_from_mapping"
8589
) as mock_get_task,
86-
patch(
87-
"nemo_evaluator_launcher.executors.slurm.executor.get_health_url"
88-
) as mock_get_health,
89-
patch(
90-
"nemo_evaluator_launcher.executors.slurm.executor.get_endpoint_url"
91-
) as mock_get_endpoint,
9290
patch(
9391
"nemo_evaluator_launcher.common.helpers.get_eval_factory_command"
9492
) as mock_get_eval_command,
@@ -103,8 +101,6 @@ def mock_dependencies(self):
103101
"endpoint_type": "openai",
104102
"task": "test_task",
105103
}
106-
mock_get_health.return_value = "http://localhost:8000/health"
107-
mock_get_endpoint.return_value = "http://localhost:8000/v1"
108104
from nemo_evaluator_launcher.common.helpers import CmdAndReadableComment
109105

110106
mock_get_eval_command.return_value = CmdAndReadableComment(
@@ -115,8 +111,6 @@ def mock_dependencies(self):
115111
yield {
116112
"load_tasks_mapping": mock_load_tasks,
117113
"get_task_from_mapping": mock_get_task,
118-
"get_health_url": mock_get_health,
119-
"get_endpoint_url": mock_get_endpoint,
120114
"get_eval_factory_command": mock_get_eval_command,
121115
"get_served_model_name": mock_get_model_name,
122116
}
@@ -594,12 +588,6 @@ def test_execute_eval_dry_run_basic(
594588
patch(
595589
"nemo_evaluator_launcher.executors.slurm.executor.get_eval_factory_command"
596590
) as mock_get_command,
597-
patch(
598-
"nemo_evaluator_launcher.executors.slurm.executor.get_health_url"
599-
) as mock_get_health,
600-
patch(
601-
"nemo_evaluator_launcher.executors.slurm.executor.get_endpoint_url"
602-
) as mock_get_endpoint,
603591
patch("builtins.print") as mock_print,
604592
):
605593
# Configure mocks
@@ -619,8 +607,6 @@ def mock_get_task_side_effect(task_name, mapping):
619607
cmd="nemo-evaluator-launcher --model llama-3.1-8b-instruct --task {task_name}",
620608
debug="# Test command for dry run",
621609
)
622-
mock_get_health.return_value = "http://localhost:8000/health"
623-
mock_get_endpoint.return_value = "http://localhost:8000/v1"
624610

625611
# Execute dry run
626612
invocation_id = SlurmExecutor.execute_eval(sample_config, dry_run=True)
@@ -738,12 +724,6 @@ def test_execute_eval_dry_run_custom_container(
738724
patch(
739725
"nemo_evaluator_launcher.executors.slurm.executor.get_eval_factory_command"
740726
) as mock_get_command,
741-
patch(
742-
"nemo_evaluator_launcher.executors.slurm.executor.get_health_url"
743-
) as mock_get_health,
744-
patch(
745-
"nemo_evaluator_launcher.executors.slurm.executor.get_endpoint_url"
746-
) as mock_get_endpoint,
747727
patch("builtins.print"),
748728
):
749729
mock_load_mapping.return_value = mock_tasks_mapping
@@ -761,8 +741,6 @@ def mock_get_task_side_effect(task_name, mapping):
761741
cmd="nemo-evaluator-launcher --task test_command",
762742
debug="# Test command for custom container",
763743
)
764-
mock_get_health.return_value = "http://localhost:8000/health"
765-
mock_get_endpoint.return_value = "http://localhost:8000/v1"
766744

767745
# Execute dry run
768746
invocation_id = SlurmExecutor.execute_eval(sample_config, dry_run=True)
@@ -800,12 +778,6 @@ def test_execute_eval_dry_run_no_auto_export(
800778
patch(
801779
"nemo_evaluator_launcher.executors.slurm.executor.get_eval_factory_command"
802780
) as mock_get_command,
803-
patch(
804-
"nemo_evaluator_launcher.executors.slurm.executor.get_health_url"
805-
) as mock_get_health,
806-
patch(
807-
"nemo_evaluator_launcher.executors.slurm.executor.get_endpoint_url"
808-
) as mock_get_endpoint,
809781
patch("builtins.print"),
810782
):
811783
mock_load_mapping.return_value = mock_tasks_mapping
@@ -823,8 +795,6 @@ def mock_get_task_side_effect(task_name, mapping):
823795
cmd="nemo-evaluator-launcher --task test_command",
824796
debug="# Test command for no auto-export",
825797
)
826-
mock_get_health.return_value = "http://localhost:8000/health"
827-
mock_get_endpoint.return_value = "http://localhost:8000/v1"
828798

829799
# Should execute successfully without auto-export
830800
invocation_id = SlurmExecutor.execute_eval(sample_config, dry_run=True)
@@ -1296,12 +1266,6 @@ def mock_subprocess_run(*args, **kwargs):
12961266
patch(
12971267
"nemo_evaluator_launcher.executors.slurm.executor.get_eval_factory_command"
12981268
) as mock_get_command,
1299-
patch(
1300-
"nemo_evaluator_launcher.executors.slurm.executor.get_health_url"
1301-
) as mock_get_health,
1302-
patch(
1303-
"nemo_evaluator_launcher.executors.slurm.executor.get_endpoint_url"
1304-
) as mock_get_endpoint,
13051269
patch("subprocess.run", side_effect=mock_subprocess_run),
13061270
):
13071271
# Configure mocks
@@ -1320,8 +1284,6 @@ def mock_get_task_side_effect(task_name, mapping):
13201284
cmd="nemo-evaluator-launcher --task mmlu_pro",
13211285
debug="# Test command for mmlu_pro",
13221286
)
1323-
mock_get_health.return_value = "http://127.0.0.1:8000/health"
1324-
mock_get_endpoint.return_value = "http://127.0.0.1:8000/v1"
13251287

13261288
# Execute non-dry-run
13271289
invocation_id = SlurmExecutor.execute_eval(sample_config, dry_run=False)
@@ -1392,12 +1354,6 @@ def mock_subprocess_run(*args, **kwargs):
13921354
patch(
13931355
"nemo_evaluator_launcher.executors.slurm.executor.get_eval_factory_command"
13941356
) as mock_get_command,
1395-
patch(
1396-
"nemo_evaluator_launcher.executors.slurm.executor.get_health_url"
1397-
) as mock_get_health,
1398-
patch(
1399-
"nemo_evaluator_launcher.executors.slurm.executor.get_endpoint_url"
1400-
) as mock_get_endpoint,
14011357
patch("subprocess.run", side_effect=mock_subprocess_run),
14021358
):
14031359
# Configure mocks
@@ -1416,8 +1372,6 @@ def mock_get_task_side_effect(task_name, mapping):
14161372
cmd="nemo-evaluator-launcher --task mmlu_pro",
14171373
debug="# Test command for mmlu_pro SSH failure",
14181374
)
1419-
mock_get_health.return_value = "http://127.0.0.1:8000/health"
1420-
mock_get_endpoint.return_value = "http://127.0.0.1:8000/v1"
14211375

14221376
# Should still succeed (SSH connection can be None)
14231377
invocation_id = SlurmExecutor.execute_eval(sample_config, dry_run=False)
@@ -1485,12 +1439,6 @@ def mock_subprocess_run(*args, **kwargs):
14851439
patch(
14861440
"nemo_evaluator_launcher.executors.slurm.executor.get_eval_factory_command"
14871441
) as mock_get_command,
1488-
patch(
1489-
"nemo_evaluator_launcher.executors.slurm.executor.get_health_url"
1490-
) as mock_get_health,
1491-
patch(
1492-
"nemo_evaluator_launcher.executors.slurm.executor.get_endpoint_url"
1493-
) as mock_get_endpoint,
14941442
patch("subprocess.run", side_effect=mock_subprocess_run),
14951443
):
14961444
# Configure mocks
@@ -1509,8 +1457,6 @@ def mock_get_task_side_effect(task_name, mapping):
15091457
cmd="nemo-evaluator-launcher --task mmlu_pro",
15101458
debug="# Test command for mmlu_pro sbatch failure",
15111459
)
1512-
mock_get_health.return_value = "http://127.0.0.1:8000/health"
1513-
mock_get_endpoint.return_value = "http://127.0.0.1:8000/v1"
15141460

15151461
# Should raise RuntimeError for sbatch failure
15161462
with pytest.raises(

0 commit comments

Comments
 (0)