feat(add-missing=-files): add missing files

Author: AWarno

packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/common/helpers.py

@@ -115,7 +115,15 @@ def apply_url_override(url: str) -> str:
         # Local executor - use localhost
         task_endpoint_type = task_definition["endpoint_type"]
         endpoint_uri = cfg.deployment.endpoints[task_endpoint_type]
-        endpoint_url = f"http://127.0.0.1:{cfg.deployment.port}{endpoint_uri}"
+
+        # Use HAProxy port if multiple_instances is enabled
+        if cfg.deployment.get("multiple_instances", False):
+            proxy_config = cfg.execution.get("proxy", {}).get("config", {})
+            port = proxy_config.get("haproxy_port", 5009)
+        else:
+            port = cfg.deployment.port
+
+        endpoint_url = f"http://127.0.0.1:{port}{endpoint_uri}"
         return endpoint_url
 
 

packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/slurm/executor.py

@@ -124,7 +124,11 @@ def execute_eval(cfg: DictConfig, dry_run: bool = False) -> str:
                 # Create HAProxy config file with placeholder IPs only if multiple_instances is true
                 if cfg.deployment.get("multiple_instances", False):
                     haproxy_config = _generate_haproxy_config_with_placeholders(cfg)
+                    # Save both template and working config
+                    haproxy_template_path = local_task_subdir / "haproxy.cfg.template"
                     haproxy_config_path = local_task_subdir / "haproxy.cfg"
+                    with open(haproxy_template_path, "w") as f:
+                        f.write(haproxy_config)
                     with open(haproxy_config_path, "w") as f:
                         f.write(haproxy_config)
 
@@ -1118,6 +1122,8 @@ def _get_proxy_server_srun_command(cfg, remote_task_subdir):
     """Generate HAProxy proxy server srun command."""
     s = ""
     s += "# HAProxy load balancer\n"
+    s += "# Copy template to config file (important for restarts)\n"
+    s += f"cp {remote_task_subdir}/haproxy.cfg.template {remote_task_subdir}/haproxy.cfg\n"
     s += "# Replace placeholder IPs with actual node IPs\n"
     s += f"haproxy_config_file={remote_task_subdir}/haproxy.cfg\n"
     s += 'for i in "${!NODES_IPS_ARRAY[@]}"; do\n'

packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/slurm/haproxy.cfg.j2

@@ -6,9 +6,9 @@ defaults
     log     global
     mode    http
     option  httplog
-    timeout connect 5s
-    timeout client  30s
-    timeout server  30s
+    timeout connect 10s
+    timeout client  3600s
+    timeout server  3600s
 
 frontend service_frontend
     bind *:{{ haproxy_port }}
@@ -18,7 +18,8 @@ backend service_backend
     mode http
     option httpchk GET {{ health_check_path }}
     http-check expect status {{ health_check_status }}
-    balance roundrobin
+    option http-server-close
+    balance leastconn
 {% for node in nodes %}
     server node{{ loop.index }} {{ node.ip }}:{{ node.port }} check
 {% endfor %}

packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/slurm/haproxy.cfg.template

@@ -0,0 +1,26 @@
+global
+    log stdout format raw local0
+    maxconn 4096
+
+defaults
+    log     global
+    mode    http
+    option  httplog
+    timeout connect 10s
+    timeout client  3600s
+    timeout server  3600s
+
+frontend service_frontend
+    bind *:{{ haproxy_port }}
+    default_backend service_backend
+
+backend service_backend
+    mode http
+    option httpchk GET {{ health_check_path }}
+    http-check expect status {{ health_check_status }}
+    option http-server-close
+    balance leastconn
+{% for node in nodes %}
+    server node{{ loop.index }} {{ node.ip }}:{{ node.port }} check
+{% endfor %}
+