GDB stub (#1583)

* gdbstub beginnings

* gdbstub: finish gdb impl things, next up is integration with melonDS

* holy fuck the gdbstub works

* gdb breakpoints work, but there's a mysterious crash on continue

* fix memory corruption that sometimes happened, and make resetting the console thru gdb work

* remove some gdb debug printing

* fix things in gdbstub

* separate option for enabling gdbstub

* add mode-dependent CPU registers

* C++ize the GDBstub code

* add gdbstub config in emu settings dialog

* make sure gdb is disabled when jit is enabled

* Remove unnecessary compiler flags, mark ARMJIT assembly code as no-execute-stack

This hardens the binary a little bit against common exploitation methods

* add option to wait for debugger attach on startup

* only insert GNU stack notes on linux

* disable gdbstub enable checkbox when jit is enabled

* fix non-linux incompatibilities

* enable gdbstub by default

* fix issues with gdbstub settings disable stuff

* format stuff

* update gdb test code

* Fix segfault when calling StubCallbacks->GetCPU()

C++ overrides are hard. Please I'm just a lowly C programmer.

* fix packet size not being sent correctly

Thanks to @GlowingUmbreon on Github for troubleshooting this

* fix select(2) calls (i should read docs more properly)

* fix GDB command sequencing/parsing issue (hopefully)

* [GDB] implement no-ack mode

* fix sending ack on handshake

* get lldb to work

7 files changed, 198 insertions, 8 deletions

diff --git a/src/frontend/qt_sdl/CMakeLists.txt b/src/frontend/qt_sdl/CMakeLists.txt
index 2426103..3923f37 100644
--- a/src/frontend/qt_sdl/CMakeLists.txt
+++ b/src/frontend/qt_sdl/CMakeLists.txt
@@ -139,6 +139,7 @@ else()
     )
     target_link_libraries(melonDS PRIVATE "${X11_LIBRARIES}" "${EGL_LIBRARIES}")
     target_include_directories(melonDS PRIVATE "${X11_INCLUDE_DIR}")
+    add_compile_definitions(QAPPLICATION_CLASS=QApplication)
 endif()
 
 
diff --git a/src/frontend/qt_sdl/Config.cpp b/src/frontend/qt_sdl/Config.cpp
index 898e4a1..da08c28 100644
--- a/src/frontend/qt_sdl/Config.cpp
+++ b/src/frontend/qt_sdl/Config.cpp
@@ -146,6 +146,14 @@ bool DSiBatteryCharging;
 
 bool DSiFullBIOSBoot;
 
+#ifdef GDBSTUB_ENABLED
+bool GdbEnabled;
+int GdbPortARM7;
+int GdbPortARM9;
+bool GdbARM7BreakOnStartup;
+bool GdbARM9BreakOnStartup;
+#endif
+
 CameraConfig Camera[2];
 
 
@@ -337,6 +345,14 @@ ConfigEntry ConfigFile[] =
 
     {"DSiFullBIOSBoot", 1, &DSiFullBIOSBoot, false, true},
 
+#ifdef GDBSTUB_ENABLED
+    {"GdbEnabled", 1, &GdbEnabled, false, false},
+    {"GdbPortARM7", 0, &GdbPortARM7, 3334, true},
+    {"GdbPortARM9", 0, &GdbPortARM9, 3333, true},
+    {"GdbARM7BreakOnStartup", 1, &GdbARM7BreakOnStartup, false, true},
+    {"GdbARM9BreakOnStartup", 1, &GdbARM9BreakOnStartup, false, true},
+#endif
+
     // TODO!!
     // we need a more elegant way to deal with this
     {"Camera0_InputType", 0, &Camera[0].InputType, 0, false},
diff --git a/src/frontend/qt_sdl/Config.h b/src/frontend/qt_sdl/Config.h
index 504c068..b1d9532 100644
--- a/src/frontend/qt_sdl/Config.h
+++ b/src/frontend/qt_sdl/Config.h
@@ -193,6 +193,12 @@ extern bool DSiFullBIOSBoot;
 
 extern CameraConfig Camera[2];
 
+extern bool GdbEnabled;
+extern int GdbPortARM7;
+extern int GdbPortARM9;
+extern bool GdbARM7BreakOnStartup;
+extern bool GdbARM9BreakOnStartup;
+
 
 void Load();
 void Save();
diff --git a/src/frontend/qt_sdl/EmuSettingsDialog.cpp b/src/frontend/qt_sdl/EmuSettingsDialog.cpp
index 0bdbb5c..571f36a 100644
--- a/src/frontend/qt_sdl/EmuSettingsDialog.cpp
+++ b/src/frontend/qt_sdl/EmuSettingsDialog.cpp
@@ -89,7 +89,22 @@ EmuSettingsDialog::EmuSettingsDialog(QWidget* parent) : QDialog(parent), ui(new
     ui->spnJITMaximumBlockSize->setDisabled(true);
 #endif
 
+#ifdef GDBSTUB_ENABLED
+    ui->cbGdbEnabled->setChecked(Config::GdbEnabled);
+    ui->intGdbPortA7->setValue(Config::GdbPortARM7);
+    ui->intGdbPortA9->setValue(Config::GdbPortARM9);
+    ui->cbGdbBOSA7->setChecked(Config::GdbARM7BreakOnStartup);
+    ui->cbGdbBOSA9->setChecked(Config::GdbARM9BreakOnStartup);
+#else
+    ui->cbGdbEnabled->setDisabled(true);
+    ui->intGdbPortA7->setDisabled(true);
+    ui->intGdbPortA9->setDisabled(true);
+    ui->cbGdbBOSA7->setDisabled(true);
+    ui->cbGdbBOSA9->setDisabled(true);
+#endif
+
     on_chkEnableJIT_toggled();
+    on_cbGdbEnabled_toggled();
     on_chkExternalBIOS_toggled();
 
     const int imgsizes[] = {256, 512, 1024, 2048, 4096, 0};
@@ -223,6 +238,12 @@ void EmuSettingsDialog::done(int r)
         bool dsiSDFolderSync = ui->cbDSiSDFolder->isChecked();
         std::string dsiSDFolderPath = ui->txtDSiSDFolder->text().toStdString();
 
+        bool gdbEnabled = ui->cbGdbEnabled->isChecked();
+        int gdbPortA7 = ui->intGdbPortA7->value();
+        int gdbPortA9 = ui->intGdbPortA9->value();
+        bool gdbBOSA7 = ui->cbGdbBOSA7->isChecked();
+        bool gdbBOSA9 = ui->cbGdbBOSA9->isChecked();
+
         if (consoleType != Config::ConsoleType
             || directBoot != Config::DirectBoot
 #ifdef JIT_ENABLED
@@ -232,6 +253,13 @@ void EmuSettingsDialog::done(int r)
             || jitLiteralOptimisations != Config::JIT_LiteralOptimisations
             || jitFastMemory != Config::JIT_FastMemory
 #endif
+#ifdef GDBSTUB_ENABLED
+            || gdbEnabled != Config::GdbEnabled
+            || gdbPortA7 != Config::GdbPortARM7
+            || gdbPortA9 != Config::GdbPortARM9
+            || gdbBOSA7 != Config::GdbARM7BreakOnStartup
+            || gdbBOSA9 != Config::GdbARM9BreakOnStartup
+#endif
             || externalBiosEnable != Config::ExternalBIOSEnable
             || bios9Path != Config::BIOS9Path
             || bios7Path != Config::BIOS7Path
@@ -285,13 +313,20 @@ void EmuSettingsDialog::done(int r)
             Config::DSiSDFolderSync = dsiSDFolderSync;
             Config::DSiSDFolderPath = dsiSDFolderPath;
 
-    #ifdef JIT_ENABLED
+#ifdef JIT_ENABLED
             Config::JIT_Enable = jitEnable;
             Config::JIT_MaxBlockSize = jitMaxBlockSize;
             Config::JIT_BranchOptimisations = jitBranchOptimisations;
             Config::JIT_LiteralOptimisations = jitLiteralOptimisations;
             Config::JIT_FastMemory = jitFastMemory;
-    #endif
+#endif
+#ifdef GDBSTUB_ENABLED
+            Config::GdbEnabled = gdbEnabled;
+            Config::GdbPortARM7 = gdbPortA7;
+            Config::GdbPortARM9 = gdbPortA9;
+            Config::GdbARM7BreakOnStartup = gdbBOSA7;
+            Config::GdbARM9BreakOnStartup = gdbBOSA9;
+#endif
 
             Config::ConsoleType = consoleType;
             Config::DirectBoot = directBoot;
@@ -506,6 +541,31 @@ void EmuSettingsDialog::on_chkEnableJIT_toggled()
         ui->chkJITFastMemory->setDisabled(disabled);
     #endif
     ui->spnJITMaximumBlockSize->setDisabled(disabled);
+
+    on_cbGdbEnabled_toggled();
+}
+
+void EmuSettingsDialog::on_cbGdbEnabled_toggled()
+{
+#ifdef GDBSTUB_ENABLED
+    bool disabled = !ui->cbGdbEnabled->isChecked();
+    bool jitenable = ui->chkEnableJIT->isChecked();
+
+    if (jitenable && !disabled) {
+        ui->cbGdbEnabled->setChecked(false);
+        disabled = true;
+    }
+#else
+    bool disabled = true;
+    bool jitenable = true;
+    ui->cbGdbEnabled->setChecked(false);
+#endif
+
+    ui->cbGdbEnabled->setDisabled(jitenable);
+    ui->intGdbPortA7->setDisabled(disabled);
+    ui->intGdbPortA9->setDisabled(disabled);
+    ui->cbGdbBOSA7->setDisabled(disabled);
+    ui->cbGdbBOSA9->setDisabled(disabled);
 }
 
 void EmuSettingsDialog::on_chkExternalBIOS_toggled()
diff --git a/src/frontend/qt_sdl/EmuSettingsDialog.h b/src/frontend/qt_sdl/EmuSettingsDialog.h
index 6a79626..2ebfd2f 100644
--- a/src/frontend/qt_sdl/EmuSettingsDialog.h
+++ b/src/frontend/qt_sdl/EmuSettingsDialog.h
@@ -77,6 +77,8 @@ private slots:
     void on_chkEnableJIT_toggled();
     void on_chkExternalBIOS_toggled();
 
+    void on_cbGdbEnabled_toggled();
+
 private:
     void verifyFirmware();
 
diff --git a/src/frontend/qt_sdl/EmuSettingsDialog.ui b/src/frontend/qt_sdl/EmuSettingsDialog.ui
index b434bbe..74bc086 100644
--- a/src/frontend/qt_sdl/EmuSettingsDialog.ui
+++ b/src/frontend/qt_sdl/EmuSettingsDialog.ui
@@ -26,7 +26,7 @@
    <item>
     <widget class="QTabWidget" name="tabWidget">
      <property name="currentIndex">
-      <number>0</number>
+      <number>5</number>
      </property>
      <widget class="QWidget" name="tab">
       <attribute name="title">
@@ -568,6 +568,101 @@
        </item>
       </layout>
      </widget>
+     <widget class="QWidget" name="tab_6">
+      <attribute name="title">
+       <string>Devtools</string>
+      </attribute>
+      <layout class="QGridLayout" name="gridLayout_3">
+       <item row="1" column="0">
+        <widget class="QLabel" name="label_16">
+         <property name="text">
+          <string>ARM9 port</string>
+         </property>
+        </widget>
+       </item>
+       <item row="5" column="0">
+        <spacer name="verticalSpacer_4">
+         <property name="orientation">
+          <enum>Qt::Vertical</enum>
+         </property>
+         <property name="sizeHint" stdset="0">
+          <size>
+           <width>20</width>
+           <height>40</height>
+          </size>
+         </property>
+        </spacer>
+       </item>
+       <item row="2" column="0">
+        <widget class="QLabel" name="label_17">
+         <property name="text">
+          <string>ARM7 port</string>
+         </property>
+        </widget>
+       </item>
+       <item row="0" column="0" colspan="3">
+        <widget class="QCheckBox" name="cbGdbEnabled">
+         <property name="text">
+          <string>Enable GDB stub</string>
+         </property>
+        </widget>
+       </item>
+       <item row="4" column="0" colspan="7">
+        <widget class="QLabel" name="label_18">
+         <property name="text">
+          <string>Note: melonDS must be restarted in order for these changes to have effect</string>
+         </property>
+        </widget>
+       </item>
+       <item row="3" column="0" colspan="7">
+        <widget class="QLabel" name="label_19">
+         <property name="text">
+          <string>Note: GDB stub cannot be used together with the JIT recompiler</string>
+         </property>
+        </widget>
+       </item>
+       <item row="1" column="6">
+        <widget class="QCheckBox" name="cbGdbBOSA9">
+         <property name="text">
+          <string>Break on startup</string>
+         </property>
+        </widget>
+       </item>
+       <item row="1" column="1" colspan="5">
+        <widget class="QSpinBox" name="intGdbPortA9">
+         <property name="minimum">
+          <number>1000</number>
+         </property>
+         <property name="maximum">
+          <number>65535</number>
+         </property>
+         <property name="value">
+          <number>3333</number>
+         </property>
+        </widget>
+       </item>
+       <item row="2" column="1" colspan="5">
+        <widget class="QSpinBox" name="intGdbPortA7">
+         <property name="minimum">
+          <number>1000</number>
+         </property>
+         <property name="maximum">
+          <number>65535</number>
+         </property>
+         <property name="value">
+          <number>3334</number>
+         </property>
+        </widget>
+       </item>
+       <item row="2" column="6">
+        <widget class="QCheckBox" name="cbGdbBOSA7">
+         <property name="text">
+          <string>Break on startup</string>
+         </property>
+        </widget>
+       </item>
+      </layout>
+     </widget>
     </widget>
    </item>
    <item>
@@ -590,7 +685,6 @@
   </customwidget>
  </customwidgets>
  <tabstops>
-  <tabstop>tabWidget</tabstop>
   <tabstop>cbxConsoleType</tabstop>
   <tabstop>chkDirectBoot</tabstop>
   <tabstop>chkExternalBIOS</tabstop>
@@ -639,8 +733,8 @@
    <slot>accept()</slot>
    <hints>
     <hint type="sourcelabel">
-     <x>257</x>
-     <y>349</y>
+     <x>266</x>
+     <y>379</y>
     </hint>
     <hint type="destinationlabel">
      <x>157</x>
@@ -655,8 +749,8 @@
    <slot>reject()</slot>
    <hints>
     <hint type="sourcelabel">
-     <x>325</x>
-     <y>349</y>
+     <x>334</x>
+     <y>379</y>
     </hint>
     <hint type="destinationlabel">
      <x>286</x>
diff --git a/src/frontend/qt_sdl/Platform.cpp b/src/frontend/qt_sdl/Platform.cpp
index 7f6e1d5..2fa0b18 100644
--- a/src/frontend/qt_sdl/Platform.cpp
+++ b/src/frontend/qt_sdl/Platform.cpp
@@ -213,6 +213,11 @@ int GetConfigInt(ConfigEntry entry)
     case Firm_Color: return Config::FirmwareFavouriteColour;
 
     case AudioBitDepth: return Config::AudioBitDepth;
+
+#ifdef GDBSTUB_ENABLED
+    case GdbPortARM7: return Config::GdbPortARM7;
+    case GdbPortARM9: return Config::GdbPortARM9;
+#endif
     }
 
     return 0;
@@ -241,6 +246,12 @@ bool GetConfigBool(ConfigEntry entry)
 
     case Firm_OverrideSettings: return Config::FirmwareOverrideSettings != 0;
     case DSi_FullBIOSBoot: return Config::DSiFullBIOSBoot != 0;
+
+#ifdef GDBSTUB_ENABLED
+    case GdbEnabled: return Config::GdbEnabled;
+    case GdbARM7BreakOnStartup: return Config::GdbARM7BreakOnStartup;
+    case GdbARM9BreakOnStartup: return Config::GdbARM9BreakOnStartup;
+#endif
     }
 
     return false;