Fix vterm resize rendering for AI sessions (#352)

Avoid installing the height-only reflow advice for vterm, and route vterm
dimension sync through its native resize handler. Flush queued vterm redraw
output during resize instead of dropping it, so Codex-style TUIs keep their
terminal state consistent without losing pending output.

Add regression coverage for vterm resize behavior and stale advice cleanup.

Author: cjf-666

ai-code-backends-infra-vterm.el

@@ -164,6 +164,24 @@ applications write to the normal screen buffer (preserving scrollback)."
              (lambda ()
                (funcall orig-fun process data)))))))))
 
+(defun ai-code-backends-infra-vterm-flush-render-queue (&optional buffer)
+  "Immediately render delayed vterm output queued for BUFFER.
+When BUFFER is nil, flush the queue for the current buffer."
+  (when (or (null buffer) (buffer-live-p buffer))
+    (with-current-buffer (or buffer (current-buffer))
+      (when ai-code-backends-infra--vterm-render-timer
+        (cancel-timer ai-code-backends-infra--vterm-render-timer))
+      (setq ai-code-backends-infra--vterm-render-timer nil)
+      (when ai-code-backends-infra--vterm-render-queue
+        (let ((data ai-code-backends-infra--vterm-render-queue))
+          (setq ai-code-backends-infra--vterm-render-queue nil)
+          (when-let* ((process (get-buffer-process (current-buffer)))
+                      ((process-live-p process)))
+            (let ((ai-code-backends-infra-vterm-anti-flicker nil))
+              (ai-code-backends-infra--vterm-render-preserving-copy-mode-view
+               (lambda ()
+                 (vterm--filter process data))))))))))
+
 (defun ai-code-backends-infra--vterm-smart-renderer (orig-fun process input)
   "Update vterm display via smart rendering around ORIG-FUN.
 Activity tracking for notifications is handled separately by

ai-code-backends-infra.el

@@ -486,9 +486,10 @@ MULTILINE-INPUT-SEQUENCE configures `S-<return>' and `C-<return>' when non-nil."
 
 ;;; Reflow and Window Management
 
-(defun ai-code-backends-infra--terminal-resize-handler ()
-  "Retrieve the terminal's resize handling function based on backend."
-  (pcase ai-code-backends-infra-terminal-backend
+(defun ai-code-backends-infra--terminal-resize-handler (&optional backend)
+  "Retrieve the resize handling function for BACKEND.
+When BACKEND is nil, use `ai-code-backends-infra-terminal-backend'."
+  (pcase (or backend ai-code-backends-infra-terminal-backend)
     ('vterm (ai-code-backends-infra-vterm-resize-handler))
     ('eat (ai-code-backends-infra-eat-resize-handler))
     ('ghostel (ai-code-backends-infra-ghostel-resize-handler))
@@ -502,7 +503,8 @@ MULTILINE-INPUT-SEQUENCE configures `S-<return>' and `C-<return>' when non-nil."
 (defun ai-code-backends-infra--terminal-reflow-filter (original-fn &rest args)
   "Filter terminal reflows to prevent height-only resize triggers.
 Suppress reflow when terminal width is unchanged or when the session
-buffer is in scroll/copy mode, working around bug #1422."
+buffer is in scroll/copy mode, working around bug #1422.
+ORIGINAL-FN and ARGS are the resize handler and arguments."
   (let* ((base-result (apply original-fn args))
          (dimensions-stable t))
     (dolist (win (window-list))
@@ -533,7 +535,7 @@ buffer is in scroll/copy mode, working around bug #1422."
   "Add or remove terminal reflow advice according to current settings."
   (let* ((resize-handler (ai-code-backends-infra--terminal-resize-handler))
          (enabled (and ai-code-backends-infra-prevent-reflow-glitch
-                       (memq ai-code-backends-infra-terminal-backend '(vterm eat)))))
+                       (eq ai-code-backends-infra-terminal-backend 'eat))))
     (dolist (handler (cl-copy-list ai-code-backends-infra--reflow-advised-handlers))
       (unless (and enabled (eq handler resize-handler))
         (when (advice-member-p #'ai-code-backends-infra--terminal-reflow-filter
@@ -542,6 +544,13 @@ buffer is in scroll/copy mode, working around bug #1422."
                          #'ai-code-backends-infra--terminal-reflow-filter))
         (setq ai-code-backends-infra--reflow-advised-handlers
               (delq handler ai-code-backends-infra--reflow-advised-handlers))))
+    (unless enabled
+      (when (advice-member-p #'ai-code-backends-infra--terminal-reflow-filter
+                             resize-handler)
+        (advice-remove resize-handler
+                       #'ai-code-backends-infra--terminal-reflow-filter))
+      (setq ai-code-backends-infra--reflow-advised-handlers
+            (delq resize-handler ai-code-backends-infra--reflow-advised-handlers)))
     (when (and enabled resize-handler)
       (unless (advice-member-p #'ai-code-backends-infra--terminal-reflow-filter
                                resize-handler)
@@ -594,9 +603,22 @@ from the window where it was initially created."
   (when (and buffer window (buffer-live-p buffer) (window-live-p window))
     (with-current-buffer buffer
       (when-let ((proc (get-buffer-process buffer)))
-        (let ((height (window-body-height window))
-              (width (window-body-width window)))
-          (set-process-window-size proc height width))))))
+        (let ((backend (ai-code-backends-infra--current-terminal-backend))
+              (windows (or (get-buffer-window-list buffer nil t)
+                           (list window))))
+          (pcase backend
+            ('vterm
+             (let ((result
+                    (funcall (ai-code-backends-infra--terminal-resize-handler
+                              'vterm)
+                             proc windows)))
+               (when result
+                 (ai-code-backends-infra-vterm-flush-render-queue buffer))
+               result))
+            (_
+             (set-process-window-size proc
+                                      (window-body-height window)
+                                      (window-body-width window)))))))))
 
 ;;; Session Helpers
 

test/test_ai-code-backends-infra.el

@@ -255,19 +255,22 @@
         (should scheduled)))))
 
 (ert-deftest test-ai-code-backends-infra-sync-reflow-filter-advice-vterm ()
-  "Enable and disable reflow advice for vterm according to toggle."
-  (let ((handler 'ai-code-backends-infra--test-resize-vterm))
+  "Do not install height-only reflow advice for vterm."
+  (let ((handler 'ai-code-backends-infra--test-resize-vterm)
+        (ai-code-backends-infra--reflow-advised-handlers nil))
     (fset handler (lambda (&rest args) args))
     (unwind-protect
         (cl-letf (((symbol-function 'ai-code-backends-infra--terminal-resize-handler)
                    (lambda () handler)))
           (let ((ai-code-backends-infra-terminal-backend 'vterm)
                 (ai-code-backends-infra-prevent-reflow-glitch t))
             (ai-code-backends-infra--sync-reflow-filter-advice)
-            (should (advice-member-p #'ai-code-backends-infra--terminal-reflow-filter
-                                     handler)))
+            (should-not (advice-member-p #'ai-code-backends-infra--terminal-reflow-filter
+                                         handler)))
+          (advice-add handler :around #'ai-code-backends-infra--terminal-reflow-filter)
+          (setq ai-code-backends-infra--reflow-advised-handlers nil)
           (let ((ai-code-backends-infra-terminal-backend 'vterm)
-                (ai-code-backends-infra-prevent-reflow-glitch nil))
+                (ai-code-backends-infra-prevent-reflow-glitch t))
             (ai-code-backends-infra--sync-reflow-filter-advice)
             (should-not (advice-member-p #'ai-code-backends-infra--terminal-reflow-filter
                                          handler))))
@@ -317,17 +320,16 @@
                        ('vterm vterm-handler)
                        ('eat eat-handler)
                        (_ (error "Unexpected backend"))))))
-          (let ((ai-code-backends-infra-terminal-backend 'vterm)
-                (ai-code-backends-infra-prevent-reflow-glitch t))
-            (ai-code-backends-infra--sync-reflow-filter-advice)
-            (should (advice-member-p #'ai-code-backends-infra--terminal-reflow-filter
-                                     vterm-handler)))
+          (advice-add vterm-handler :around #'ai-code-backends-infra--terminal-reflow-filter)
+          (setq ai-code-backends-infra--reflow-advised-handlers (list vterm-handler))
           (let ((ai-code-backends-infra-terminal-backend 'eat)
                 (ai-code-backends-infra-prevent-reflow-glitch t)
                 (ai-code-backends-infra-eat-preserve-position nil))
             (ai-code-backends-infra--sync-reflow-filter-advice))
           (should-not (advice-member-p #'ai-code-backends-infra--terminal-reflow-filter
-                                       vterm-handler)))
+                                       vterm-handler))
+          (should (advice-member-p #'ai-code-backends-infra--terminal-reflow-filter
+                                   eat-handler)))
       (dolist (handler (list vterm-handler eat-handler))
         (when (advice-member-p #'ai-code-backends-infra--terminal-reflow-filter handler)
           (advice-remove handler #'ai-code-backends-infra--terminal-reflow-filter))
@@ -357,6 +359,126 @@
         (funcall (cdr (assq 'window-width captured-entry)) 'fake-window)
         (should (equal resize-call '(fake-window 4 t)))))))
 
+(ert-deftest test-ai-code-backends-infra-terminal-reflow-filter-ignores-non-ai-vterm-buffer ()
+  "The reflow filter should pass through non-session vterm buffers."
+  (with-temp-buffer
+    (rename-buffer "*vterm*" t)
+    (let ((ai-code-backends-infra-terminal-backend 'vterm)
+          (ai-code-backends-infra-prevent-reflow-glitch t)
+          (original-called nil))
+      (cl-letf (((symbol-function 'window-list)
+                 (lambda (&rest _args) (list 'fake-window)))
+                ((symbol-function 'window-buffer)
+                 (lambda (_window) (current-buffer))))
+        (should (eq (ai-code-backends-infra--terminal-reflow-filter
+                     (lambda (&rest _args)
+                       (setq original-called t)
+                       'native-result)
+                     'fake-arg)
+                    'native-result))
+        (should original-called)))))
+
+(ert-deftest test-ai-code-backends-infra-sync-terminal-dimensions-vterm-height-change ()
+  "Vterm height changes should go through the native resize handler."
+  (let* ((buffer (get-buffer-create "*codex[vterm-height-resize]*"))
+         (process (start-process "ai-code-vterm-resize-test"
+                                 buffer
+                                 "sleep"
+                                 "5"))
+         (window 'fake-window)
+         (height 24)
+         (width 80)
+         resize-calls
+         rendered
+         cancelled-timer)
+    (unwind-protect
+        (cl-letf (((symbol-function 'window-live-p)
+                   (lambda (candidate) (eq candidate window)))
+                  ((symbol-function 'get-buffer-window-list)
+                   (lambda (_buffer &rest _args) (list window)))
+                  ((symbol-function 'window-body-height)
+                   (lambda (_window) height))
+                  ((symbol-function 'window-body-width)
+                   (lambda (_window) width))
+                  ((symbol-function 'set-process-window-size)
+                   (lambda (&rest _args)
+                     (ert-fail "vterm sync should use the native resize handler")))
+                  ((symbol-function 'ai-code-backends-infra--terminal-resize-handler)
+                   (lambda (&optional backend)
+                     (should (eq backend 'vterm))
+                     (lambda (proc windows)
+                       (push (list proc windows height width) resize-calls)
+                       (cons width height))))
+                  ((symbol-function 'cancel-timer)
+                   (lambda (timer) (setq cancelled-timer timer)))
+                  ((symbol-function 'vterm--filter)
+                   (lambda (proc data)
+                     (push (list proc data ai-code-backends-infra-vterm-anti-flicker)
+                           rendered))))
+          (with-current-buffer buffer
+            (setq-local ai-code-backends-infra--session-terminal-backend 'vterm)
+            (setq-local ai-code-backends-infra--vterm-render-timer 'mock-timer)
+            (setq-local ai-code-backends-infra--vterm-render-queue "old-redraw-1")
+            (ai-code-backends-infra--sync-terminal-dimensions buffer window)
+            (setq height 18)
+            (setq-local ai-code-backends-infra--vterm-render-queue "old-redraw-2")
+            (ai-code-backends-infra--sync-terminal-dimensions buffer window)
+            (should (null ai-code-backends-infra--vterm-render-timer))
+            (should-not ai-code-backends-infra--vterm-render-queue))
+          (should (eq cancelled-timer 'mock-timer))
+          (should (equal (nreverse resize-calls)
+                         `((,process (,window) 24 80)
+                           (,process (,window) 18 80)))))
+          (should (equal (nreverse rendered)
+                         `((,process "old-redraw-1" nil)
+                           (,process "old-redraw-2" nil))))
+      (when (process-live-p process)
+        (delete-process process))
+      (when (buffer-live-p buffer)
+        (kill-buffer buffer)))))
+
+(ert-deftest test-ai-code-backends-infra-sync-terminal-dimensions-vterm-width-change ()
+  "Vterm width changes should go through the native resize handler."
+  (let* ((buffer (get-buffer-create "*codex[vterm-width-resize]*"))
+         (process (start-process "ai-code-vterm-width-resize-test"
+                                 buffer
+                                 "sleep"
+                                 "5"))
+         (window 'fake-window)
+         (height 24)
+         (width 80)
+         calls)
+    (unwind-protect
+        (cl-letf (((symbol-function 'window-live-p)
+                   (lambda (candidate) (eq candidate window)))
+                  ((symbol-function 'get-buffer-window-list)
+                   (lambda (_buffer &rest _args) (list window)))
+                  ((symbol-function 'window-body-height)
+                   (lambda (_window) height))
+                  ((symbol-function 'window-body-width)
+                   (lambda (_window) width))
+                  ((symbol-function 'set-process-window-size)
+                   (lambda (&rest _args)
+                     (ert-fail "vterm sync should not call generic process sizing")))
+                  ((symbol-function 'ai-code-backends-infra--terminal-resize-handler)
+                   (lambda (&optional backend)
+                     (should (eq backend 'vterm))
+                     (lambda (proc windows)
+                       (push (list proc windows height width) calls)
+                       (cons width height)))))
+          (with-current-buffer buffer
+            (setq-local ai-code-backends-infra--session-terminal-backend 'vterm)
+            (ai-code-backends-infra--sync-terminal-dimensions buffer window)
+            (setq width 100)
+            (ai-code-backends-infra--sync-terminal-dimensions buffer window))
+          (should (equal (nreverse calls)
+                         `((,process (,window) 24 80)
+                           (,process (,window) 24 100)))))
+      (when (process-live-p process)
+        (delete-process process))
+      (when (buffer-live-p buffer)
+        (kill-buffer buffer)))))
+
 (ert-deftest test-ai-code-backends-infra-sync-terminal-cursor-vterm-copy-mode ()
   "Show an Emacs cursor in vterm copy mode and restore terminal cursor on exit."
   (with-temp-buffer