Offload traffic to static workers and use node as the proxy

.changeset/wild-hounds-laugh.md

@@ -0,0 +1,6 @@
+---
+"@self/app": minor
+"gradio": minor
+---
+
+feat:Offload traffic to static workers and use node as the proxy

gradio/blocks.py

@@ -610,6 +610,22 @@ def convert_component_dict_to_list(
     return predictions
 
 
+def _find_free_port(host: str, start: int, try_count: int = 100) -> int:
+    """Find an available port by scanning from *start*."""
+    import socket
+
+    for port in range(start, start + try_count):
+        try:
+            s = socket.socket()
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            s.bind((host if host != "0.0.0.0" else "127.0.0.1", port))
+            s.close()
+            return port
+        except OSError:
+            continue
+    raise OSError(f"Cannot find empty port in range: {start}-{start + try_count - 1}.")
+
+
 class BlocksConfig:
     def __init__(self, root_block: Blocks):
         self._id: int = 0
@@ -2544,6 +2560,7 @@ def launch(
         ssr_mode: bool | None = None,
         pwa: bool | None = None,
         mcp_server: bool | None = None,
+        num_workers: int | None = None,
         _app: App | None = None,
         _frontend: bool = True,
         i18n: I18n | None = None,
@@ -2726,15 +2743,77 @@ def reverse(text):
         self.config = self.get_config_file()
 
         self.ssr_mode = self._resolve_ssr_mode(ssr_mode, disable_when_multi_page=False)
+
+        # Resolve num_workers early so we can pre-compute worker ports
+        # and pass them to the Node proxy at startup.
+        resolved_num_workers = num_workers
+        if resolved_num_workers is None:
+            env_val = os.environ.get("GRADIO_NUM_WORKERS")
+            if env_val is not None:
+                resolved_num_workers = int(env_val)
+
+        self._node_is_proxy = False
+        static_worker_ports: list[int] = []
+
         if self.ssr_mode:
             self.node_path = os.environ.get("GRADIO_NODE_PATH", get_node_path())
-            self.node_server_name, self.node_process, self.node_port = (
-                start_node_server(
-                    server_name=node_server_name,
-                    server_port=node_port,
-                    node_path=self.node_path,
+            is_dev_mode = os.getenv("GRADIO_LOCAL_DEV_MODE") is not None
+
+            if is_dev_mode:
+                # Dev mode: vite dev server on 9876, Python is the front.
+                # Keep the old architecture (Python proxies to vite).
+                self.node_server_name, self.node_process, self.node_port = (
+                    start_node_server(
+                        server_name=node_server_name,
+                        server_port=node_port,
+                        node_path=self.node_path,
+                    )
                 )
-            )
+            else:
+                # Production: Node is the front proxy on the user-facing port.
+                # Python moves to an internal port after Node.
+                # Workers get ports after Python.
+                from gradio.http_server import INITIAL_PORT_VALUE, TRY_NUM_PORTS
+
+                user_port = server_port or int(
+                    os.getenv("GRADIO_SERVER_PORT", str(INITIAL_PORT_VALUE))
+                )
+                python_host = server_name or os.getenv(
+                    "GRADIO_SERVER_NAME", "127.0.0.1"
+                )
+
+                # Find a free port for Python starting after the user-facing port.
+                # We need to know Python's port before starting Node so Node
+                # can proxy to it.
+                python_internal_port = _find_free_port(
+                    python_host, start=user_port + 1, try_count=TRY_NUM_PORTS
+                )
+
+                # Pre-compute static worker ports
+                if resolved_num_workers is not None and resolved_num_workers >= 1:
+                    worker_start = python_internal_port + 1
+                    static_worker_ports = [
+                        worker_start + i for i in range(resolved_num_workers)
+                    ]
+
+                self.node_server_name, self.node_process, self.node_port = (
+                    start_node_server(
+                        server_name=node_server_name or python_host,
+                        server_port=node_port or user_port,
+                        node_path=self.node_path,
+                        python_port=python_internal_port,
+                        python_host=python_host,
+                        static_worker_ports=static_worker_ports,
+                    )
+                )
+                # Only use the proxy architecture if Node actually started
+                if self.node_process is not None and self.node_port is not None:
+                    server_port = python_internal_port
+                    self._node_is_proxy = True
+                else:
+                    # Node failed to start — fall back to Python as the
+                    # front server on the original user port.
+                    static_worker_ports = []
         else:
             self.node_server_name = self.node_port = self.node_process = None
 
@@ -2796,21 +2875,74 @@ def reverse(text):
             if self.mcp_server_obj:
                 self.mcp_server_obj._local_url = self.local_url
 
+            # When Node is the front proxy, the user-facing URL is the Node port
+            if self._node_is_proxy and self.node_port is not None:
+                url_host = (
+                    "localhost" if self.server_name == "0.0.0.0" else self.server_name
+                )
+                self.local_url = f"http://{url_host}:{self.node_port}/"
+                self.local_api_url = f"{self.local_url.rstrip('/')}{API_PREFIX}/"
+
             self.protocol = (
                 "https"
                 if self.local_url.startswith("https") or self.is_colab
                 else "http"
             )
             if not self.is_colab and not quiet:
-                s = (
-                    "* Running on local URL:  {}://{}:{}, with SSR ⚡ (experimental, to disable set `ssr_mode=False` in `launch()`)"
-                    if self.ssr_mode
-                    else "* Running on local URL:  {}://{}:{}"
-                )
-                print(s.format(self.protocol, self.server_name, self.server_port))
+                if self._node_is_proxy and self.node_port is not None:
+                    print(
+                        f"* Running on local URL:  {self.protocol}://{self.server_name}:{self.node_port}, with SSR ⚡ (Node proxy -> Python :{self.server_port})"
+                    )
+                elif self.ssr_mode:
+                    print(
+                        f"* Running on local URL:  {self.protocol}://{self.server_name}:{self.server_port}, with SSR ⚡ (dev mode)"
+                    )
+                else:
+                    s = "* Running on local URL:  {}://{}:{}"
+                    print(s.format(self.protocol, self.server_name, self.server_port))
 
             self._queue.set_server_app(self.server_app)
 
+            # Static worker pool for offloading file serving / uploads
+            if resolved_num_workers is not None and resolved_num_workers >= 1:
+                from gradio.routes import (
+                    BUILD_PATH_LIB,
+                    STATIC_PATH_LIB,
+                )
+                from gradio.static_server import StaticServerConfig, StaticWorkerPool
+
+                static_config = StaticServerConfig(
+                    build_path=str(BUILD_PATH_LIB),
+                    static_path=str(STATIC_PATH_LIB),
+                    uploaded_file_dir=self.server_app.uploaded_file_dir,
+                    allowed_paths=self.allowed_paths,
+                    blocked_paths=self.blocked_paths,
+                    max_file_size=self.max_file_size,
+                    favicon_path=str(self.favicon_path) if self.favicon_path else None,
+                )
+                # When Node is the front proxy, worker ports were pre-computed above.
+                # Otherwise, compute them here.
+                if self._node_is_proxy and static_worker_ports:
+                    worker_ports = static_worker_ports
+                else:
+                    worker_start = self.server_port + 1
+                    if self.node_port is not None:
+                        worker_start = max(worker_start, self.node_port + 1)
+                    worker_ports = [
+                        worker_start + i for i in range(resolved_num_workers)
+                    ]
+                self._static_worker_pool = StaticWorkerPool(
+                    num_workers=resolved_num_workers,
+                    config=static_config,
+                    ports=worker_ports,
+                )
+                self._static_worker_pool.start()
+
+                if not quiet:
+                    print(
+                        f"* Static file workers: {resolved_num_workers} processes on ports {self._static_worker_pool.ports}"
+                    )
+
             resp = httpx.get(
                 f"{self.local_api_url}startup-events",
                 verify=ssl_verify,
@@ -3086,6 +3218,12 @@ def close(self, verbose: bool = True) -> None:
             return
 
         try:
+            if (
+                hasattr(self, "_static_worker_pool")
+                and self._static_worker_pool is not None
+            ):
+                self._static_worker_pool.shutdown()
+                self._static_worker_pool = None
             self._queue.close()
             # set this before closing server to shut down heartbeats
             self.is_running = False

gradio/node_server.py

@@ -29,13 +29,19 @@ def start_node_server(
     server_name: str | None = None,
     server_port: int | None = None,
     node_path: str | None = None,
+    python_port: int | None = None,
+    python_host: str | None = None,
+    static_worker_ports: list[int] | None = None,
 ) -> tuple[str | None, subprocess.Popen[bytes] | None, int | None]:
-    """Launches a local server running the provided Interface
+    """Launches the Node SSR server as a front proxy.
+
     Parameters:
         server_name: to make app accessible on local network, set this to "0.0.0.0". Can be set by environment variable GRADIO_SERVER_NAME.
         server_port: will start gradio app on this port (if available). Can be set by environment variable GRADIO_SERVER_PORT.
         node_path: the path to the node executable. Can be set by environment variable GRADIO_NODE_PATH.
-        ssr_mode: If False, will not start the node server and will serve the SPA from the Python server
+        python_port: the port of the main Python (FastAPI) server that Node will proxy to.
+        python_host: the host of the main Python server (default 127.0.0.1).
+        static_worker_ports: ports of static file worker processes for round-robin proxying.
 
     Returns:
         server_name: the name of the server (default is "localhost")
@@ -56,13 +62,16 @@ def start_node_server(
     server_ports = (
         [server_port]
         if server_port is not None
-        else range(INITIAL_PORT_VALUE + 1, INITIAL_PORT_VALUE + 1 + TRY_NUM_PORTS)
+        else range(INITIAL_PORT_VALUE, INITIAL_PORT_VALUE + TRY_NUM_PORTS)
     )
 
     node_process, node_port = start_node_process(
         node_path=node_path or os.getenv("GRADIO_NODE_PATH"),
         server_name=host,
         server_ports=server_ports,
+        python_port=python_port,
+        python_host=python_host or "127.0.0.1",
+        static_worker_ports=static_worker_ports or [],
     )
 
     return server_name, node_process, node_port
@@ -76,6 +85,9 @@ def start_node_process(
     node_path: str | None,
     server_name: str,
     server_ports: list[int] | range,
+    python_port: int | None = None,
+    python_host: str = "127.0.0.1",
+    static_worker_ports: list[int] | None = None,
 ) -> tuple[subprocess.Popen[bytes] | None, int | None]:
     if GRADIO_LOCAL_DEV_MODE:
         return None, 9876
@@ -103,6 +115,15 @@ def start_node_process(
             if GRADIO_LOCAL_DEV_MODE:
                 env["GRADIO_LOCAL_DEV_MODE"] = "1"
 
+            # Proxy configuration: tell Node where Python and workers are
+            if python_port is not None:
+                env["GRADIO_PYTHON_PORT"] = str(python_port)
+                env["GRADIO_PYTHON_HOST"] = python_host
+            if static_worker_ports:
+                env["GRADIO_STATIC_WORKER_PORTS"] = ",".join(
+                    str(p) for p in static_worker_ports
+                )
+
             register_file = str(
                 Path(__file__).parent.joinpath("templates", "register.mjs")
             )
@@ -112,11 +133,14 @@ def start_node_process(
 
             node_process = subprocess.Popen(
                 [node_path, "--import", register_file, SSR_APP_PATH],
-                stdout=subprocess.DEVNULL,
                 env=env,
             )
 
-            is_working = verify_server_startup(server_name, port, timeout=5)
+            # When Node is the front proxy, Python isn't up yet so SSR
+            # pages will fail. Just check TCP connectivity.
+            is_working = verify_server_startup(
+                server_name, port, timeout=5, tcp_only=(python_port is not None)
+            )
             if is_working:
                 signal.signal(
                     signal.SIGTERM, lambda _, __: handle_sigterm(node_process)
@@ -167,23 +191,29 @@ def attempt_connection(host: str, port: int) -> bool:
         return False
 
 
-def verify_server_startup(host: str, port: int, timeout: float = 15.0) -> bool:
-    """Verifies if a server is up and running by making an HTTP request.
+def verify_server_startup(
+    host: str, port: int, timeout: float = 15.0, tcp_only: bool = False
+) -> bool:
+    """Verifies if a server is up and running.
 
-    A simple TCP connection check is not sufficient because the Node SSR server
-    may accept connections before it is ready to handle HTTP requests. This
-    would cause Gradio's own url_ok health check (which does HEAD /) to fail
-    intermittently.
+    When tcp_only=False (default), makes an HTTP HEAD request and checks for
+    a non-500 status. When tcp_only=True, only checks TCP connectivity — this
+    is needed when Node acts as a front proxy because the SSR handler cannot
+    render pages until the Python backend is up.
     """
     start_time = time.time()
     while time.time() - start_time < timeout:
         try:
-            conn = HTTPConnection(host, port, timeout=2)
-            conn.request("HEAD", "/")
-            resp = conn.getresponse()
-            conn.close()
-            if resp.status < 500:
-                return True
+            if tcp_only:
+                with closing(socket.create_connection((host, port), timeout=2)):
+                    return True
+            else:
+                conn = HTTPConnection(host, port, timeout=2)
+                conn.request("HEAD", "/")
+                resp = conn.getresponse()
+                conn.close()
+                if resp.status < 500:
+                    return True
         except Exception:
             pass
         time.sleep(0.2)