GdbServer: Rework sleeping

Source/Tools/FEXLoader/FEXLoader.cpp

@@ -587,16 +587,17 @@ int main(int argc, char** argv, char** const envp) {
   fextl::unique_ptr<FEX::GdbServer> DebugServer;
   if (GdbServer) {
     DebugServer = fextl::make_unique<FEX::GdbServer>(CTX.get(), SignalDelegation.get(), SyscallHandler.get());
+    SyscallHandler->SetGdbServer(DebugServer.get());
   }
 
   if (!CTX->InitCore()) {
     return 1;
   }
 
   auto ParentThread = SyscallHandler->TM.CreateThread(Loader.DefaultRIP(), Loader.GetStackPointer());
-  SyscallHandler->TM.TrackThread(ParentThread);
   SignalDelegation->RegisterTLSState(ParentThread);
   ThunkHandler->RegisterTLSState(ParentThread);
+  SyscallHandler->TM.TrackThread(ParentThread);
 
   // Pass in our VDSO thunks
   ThunkHandler->AppendThunkDefinitions(FEX::VDSO::GetVDSOThunkDefinitions(Loader.Is64BitMode()));

Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.cpp

@@ -10,6 +10,7 @@ desc: Provides a gdb interface to the guest state
 #include "GdbServer/Info.h"
 
 #include "LinuxSyscalls/NetStream.h"
+#include "LinuxSyscalls/SignalDelegator.h"
 
 #include <cstdlib>
 #include <cstdio>
@@ -62,25 +63,19 @@ desc: Provides a gdb interface to the guest state
 namespace FEX {
 
 #ifndef _WIN32
-void GdbServer::Break(FEXCore::Core::InternalThreadState* Thread, int signal) {
+void GdbServer::Break(FEX::HLE::ThreadStateObject* ThreadObject, int signal) {
   std::lock_guard lk(sendMutex);
   if (!CommsStream) {
     return;
   }
 
-  auto ThreadObject = FEX::HLE::ThreadManager::GetStateObjectFromFEXCoreThread(Thread);
   // Current debugging thread switches to the thread that is breaking.
   CurrentDebuggingThread = ThreadObject->ThreadInfo.TID.load();
 
   const auto str = fextl::fmt::format("T{:02x}thread:{:x};", signal, CurrentDebuggingThread);
   SendPacket(*CommsStream, str);
 }
 
-void GdbServer::WaitForThreadWakeup() {
-  // Wait for gdbserver to tell us to wake up
-  ThreadBreakEvent.Wait();
-}
-
 GdbServer::~GdbServer() {
   CloseListenSocket();
   CoreShuttingDown = true;
@@ -119,9 +114,9 @@ GdbServer::GdbServer(FEXCore::Context::Context* ctx, FEX::HLE::SignalDelegator*
       ThreadObject->GdbInfo->PState = ArchHelpers::Context::GetArmPState(ucontext);
 
       // Let GDB know that we have a signal
-      this->Break(Thread, Signal);
+      this->Break(ThreadObject, Signal);
 
-      WaitForThreadWakeup();
+      this->SyscallHandler->TM.SleepThread(this->CTX, ThreadObject);
       ThreadObject->GdbInfo.reset();
 
       return true;
@@ -605,8 +600,14 @@ GdbServer::HandledPacketType GdbServer::handleProgramOffsets() {
 GdbServer::HandledPacketType GdbServer::ThreadAction(char action, uint32_t tid) {
   switch (action) {
   case 'c': {
+    {
+      std::lock_guard lk(SyscallHandler->TM.GetThreadsCreationMutex());
+      auto Threads = SyscallHandler->TM.GetThreads();
+      for (auto& Thread : *Threads) {
+        Thread->ThreadSleeping.NotifyOne();
+      }
+    }
     SyscallHandler->TM.Run();
-    ThreadBreakEvent.NotifyAll();
     SyscallHandler->TM.WaitForThreadsToRun();
     return {"", HandledPacketType::TYPE_ONLYACK};
   }
@@ -1451,6 +1452,13 @@ void GdbServer::StartThread() {
   FEXCore::Threads::SetSignalMask(OldMask);
 }
 
+void GdbServer::OnThreadCreated(FEX::HLE::ThreadStateObject* ThreadObject) {
+  if (SyscallHandler->TM.GetThreadCount() == 1) {
+    // Sleep the first thread created. This is because FEX only supports attaching at startup currently.
+    SyscallHandler->TM.SleepThread(CTX, ThreadObject);
+  }
+}
+
 void GdbServer::OpenListenSocket() {
   const auto GdbUnixPath = fextl::fmt::format("{}/FEX_gdbserver/", FEXServerClient::GetTempFolder());
   if (FHU::Filesystem::CreateDirectory(GdbUnixPath) == FHU::Filesystem::CreateDirectoryResult::ERROR) {

Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.h

@@ -35,8 +35,10 @@ class GdbServer {
     LibraryMapChanged = true;
   }
 
+  void OnThreadCreated(FEX::HLE::ThreadStateObject* ThreadObject);
+
 private:
-  void Break(FEXCore::Core::InternalThreadState* Thread, int signal);
+  void Break(FEX::HLE::ThreadStateObject* ThreadObject, int signal);
 
   void OpenListenSocket();
   void CloseListenSocket();
@@ -52,9 +54,6 @@ class GdbServer {
 
   void SendACK(std::ostream& stream, bool NACK);
 
-  Event ThreadBreakEvent {};
-  void WaitForThreadWakeup();
-
   struct HandledPacketType {
     fextl::string Response {};
     enum ResponseType {

Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp

@@ -476,7 +476,7 @@ bool SignalDelegator::HandleSignalPause(FEXCore::Core::InternalThreadState* Thre
     // We need to be a little bit careful here
     // If we were already paused (due to GDB) and we are immediately stopping (due to gdb kill)
     // Then we need to ensure we don't double decrement our idle thread counter
-    if (ThreadObject->ThreadSleeping) {
+    if (ThreadObject->ThreadSleeping.HasWaiter()) {
       // If the thread was sleeping then its idle counter was decremented
       // Reincrement it here to not break logic
       FEX::HLE::_SyscallHandler->TM.IncrementIdleRefCount();

Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.cpp

@@ -744,7 +744,7 @@ void SyscallHandler::DefaultProgramBreak(uint64_t Base, uint64_t Size) {
 }
 
 SyscallHandler::SyscallHandler(FEXCore::Context::Context* _CTX, FEX::HLE::SignalDelegator* _SignalDelegation, FEX::HLE::ThunkHandler* ThunkHandler)
-  : TM {_CTX, _SignalDelegation}
+  : TM {_CTX, this, _SignalDelegation}
   , SeccompEmulator {this, _SignalDelegation}
   , FM {_CTX}
   , CTX {_CTX}

Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h

@@ -53,7 +53,8 @@ desc: Glue logic, STRACE magic
 
 namespace FEX {
 class CodeLoader;
-}
+class GdbServer;
+} // namespace FEX
 
 namespace FEXCore {
 namespace Context {
@@ -282,6 +283,14 @@ class SyscallHandler : public FEXCore::HLE::SyscallHandler, FEXCore::HLE::Source
 
   constexpr static uint64_t TASK_MAX_64BIT = (1ULL << 48);
 
+  void SetGdbServer(FEX::GdbServer* Server) {
+    GdbServer = Server;
+  }
+
+  FEX::GdbServer* GetGdbServer() {
+    return GdbServer;
+  }
+
 protected:
   SyscallHandler(FEXCore::Context::Context* _CTX, FEX::HLE::SignalDelegator* _SignalDelegation, FEX::HLE::ThunkHandler* ThunkHandler);
 
@@ -308,6 +317,7 @@ class SyscallHandler : public FEXCore::HLE::SyscallHandler, FEXCore::HLE::Source
 
   FEX::HLE::SignalDelegator* SignalDelegation;
   FEX::HLE::ThunkHandler* ThunkHandler;
+  FEX::GdbServer* GdbServer {};
 
   std::mutex FutexMutex;
   std::mutex SyscallMutex;

Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp

@@ -230,12 +230,12 @@ uint64_t HandleNewClone(FEX::HLE::ThreadStateObject* Thread, FEXCore::Context::C
   Thread->ThreadInfo.PID = ::getpid();
   FEX::HLE::_SyscallHandler->FM.UpdatePID(Thread->ThreadInfo.PID);
 
+  FEX::HLE::_SyscallHandler->RegisterTLSState(Thread);
+
   if (CreatedNewThreadObject) {
     FEX::HLE::_SyscallHandler->TM.TrackThread(Thread);
   }
 
-  FEX::HLE::_SyscallHandler->RegisterTLSState(Thread);
-
   // Start exuting the thread directly
   // Our host clone starts in a new stack space, so it can't return back to the JIT space
   CTX->ExecuteThread(Thread->Thread);

Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.cpp

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: MIT
 
+#include "LinuxSyscalls/GdbServer.h"
 #include "LinuxSyscalls/Syscalls.h"
 #include "LinuxSyscalls/SignalDelegator.h"
 
@@ -29,6 +30,18 @@ FEX::HLE::ThreadStateObject* ThreadManager::CreateThread(uint64_t InitialRIP, ui
   return ThreadStateObject;
 }
 
+void ThreadManager::TrackThread(FEX::HLE::ThreadStateObject* Thread) {
+  {
+    std::lock_guard lk(ThreadCreationMutex);
+    Threads.emplace_back(Thread);
+  }
+
+  auto GdbServer = SyscallHandler->GetGdbServer();
+  if (GdbServer) {
+    GdbServer->OnThreadCreated(Thread);
+  }
+}
+
 void ThreadManager::DestroyThread(FEX::HLE::ThreadStateObject* Thread, bool NeedsTLSUninstall) {
   {
     std::lock_guard lk(ThreadCreationMutex);
@@ -79,7 +92,10 @@ void ThreadManager::NotifyPause() {
   // Tell all the threads that they should pause
   std::lock_guard lk(ThreadCreationMutex);
   for (auto& Thread : Threads) {
-    SignalDelegation->SignalThread(Thread->Thread, SignalEvent::Pause);
+    if (!Thread->ThreadSleeping.HasWaiter()) {
+      // Only signal if it isn't already sleeping.
+      SignalDelegation->SignalThread(Thread->Thread, SignalEvent::Pause);
+    }
   }
 }
 
@@ -182,8 +198,7 @@ void ThreadManager::Stop(bool IgnoreCurrentThread) {
   }
 }
 
-void ThreadManager::SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame) {
-  auto ThreadObject = FEX::HLE::ThreadManager::GetStateObjectFromCPUState(Frame);
+void ThreadManager::SleepThread(FEXCore::Context::Context* CTX, FEX::HLE::ThreadStateObject* ThreadObject) {
 #if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
   // Sanity check. This can only be called from the owning thread.
   {
@@ -197,19 +212,16 @@ void ThreadManager::SleepThread(FEXCore::Context::Context* CTX, FEXCore::Core::C
   --IdleWaitRefCount;
   IdleWaitCV.notify_all();
 
-  ThreadObject->ThreadSleeping = true;
-
-  // Go to sleep
-  ThreadObject->ThreadPaused.Wait();
+  // Go to sleep.
+  ThreadObject->ThreadSleeping.Wait();
 
   ++IdleWaitRefCount;
-  ThreadObject->ThreadSleeping = false;
 
   IdleWaitCV.notify_all();
 }
 
 void ThreadManager::UnpauseThread(FEX::HLE::ThreadStateObject* Thread) {
-  Thread->ThreadPaused.NotifyOne();
+  Thread->ThreadSleeping.NotifyOne();
 }
 
 void ThreadManager::UnlockAfterFork(FEXCore::Core::InternalThreadState* LiveThread, bool Child) {