Detect state cycles in P_SetMobjState()

Fixes crash in MAP10 of Dominus Diabolicus 2. Thanks bofu! https://www.doomworld.com/forum/post/2920133

Author: bradharding

src/p_mobj.c

@@ -58,14 +58,26 @@
 //
 bool P_SetMobjState(mobj_t *mobj, statenum_t state)
 {
+    // killough 4/9/98: remember states seen, to detect cycles:
+
+    // fast transition table
+    statenum_t  *seenstate = seenstate_tab;     // pointer to table
+    static int  recursion;                      // detects recursion
+    statenum_t  i = state;                      // initial state
+    boolean     ret = true;                     // return value
+    statenum_t  *tempstate = NULL;              // for use with recursion
+
+    if (recursion++)                            // if recursion detected, allocate state table
+        seenstate = tempstate = Z_Calloc(numstates, sizeof(statenum_t), PU_STATIC, NULL);
+
     do
     {
         if (state == S_NULL)
         {
             mobj->state = (state_t *)S_NULL;
             P_RemoveMobj(mobj);
-
-            return false;
+            ret = false;
+            break;                              // killough 4/9/98
         }
         else
         {
@@ -81,11 +93,19 @@ bool P_SetMobjState(mobj_t *mobj, statenum_t state)
             if (st->action)
                 st->action(mobj, NULL, NULL);
 
+            seenstate[state] = 1 + st->nextstate;   // killough 4/9/98
             state = st->nextstate;
         }
-    } while (!mobj->tics);
+    } while (!mobj->tics && !seenstate[state]); // killough 4/9/98
 
-    return true;
+    if (!--recursion)
+        for (; (state = seenstate[i]); i = state - 1)
+            seenstate[i] = 0;                   // killough 4/9/98: erase memory of states
+
+    if (tempstate)
+        Z_Free(tempstate);
+
+    return ret;
 }
 
 //

src/states.c

@@ -1509,14 +1509,18 @@ state_t original_states[] =
 // DSDHacked
 state_t     *states;
 int         numstates;
-byte        *defined_codeptr_args;
+byte        *defined_codeptr_args = NULL;
+statenum_t  *seenstate_tab = NULL;
 actionf_t   *deh_codeptr;
 
 void InitStates(void)
 {
     numstates = NUMSTATES;
     states = original_states;
 
+    array_grow(seenstate_tab, numstates);
+    memset(seenstate_tab, 0, numstates * sizeof(*seenstate_tab));
+
     array_grow(deh_codeptr, numstates);
 
     for (int i = 0; i < numstates; i++)

src/states.h

@@ -1179,6 +1179,7 @@ extern state_t      *states;
 extern int          numstates;
 extern actionf_t    *deh_codeptr;
 extern byte         *defined_codeptr_args;
+extern statenum_t   *seenstate_tab;
 
 void InitStates(void);
 void FreeStates(void);