GDB stub (#1583)

* gdbstub beginnings

* gdbstub: finish gdb impl things, next up is integration with melonDS

* holy fuck the gdbstub works

* gdb breakpoints work, but there's a mysterious crash on continue

* fix memory corruption that sometimes happened, and make resetting the console thru gdb work

* remove some gdb debug printing

* fix things in gdbstub

* separate option for enabling gdbstub

* add mode-dependent CPU registers

* C++ize the GDBstub code

* add gdbstub config in emu settings dialog

* make sure gdb is disabled when jit is enabled

* Remove unnecessary compiler flags, mark ARMJIT assembly code as no-execute-stack

This hardens the binary a little bit against common exploitation methods

* add option to wait for debugger attach on startup

* only insert GNU stack notes on linux

* disable gdbstub enable checkbox when jit is enabled

* fix non-linux incompatibilities

* enable gdbstub by default

* fix issues with gdbstub settings disable stuff

* format stuff

* update gdb test code

* Fix segfault when calling StubCallbacks->GetCPU()

C++ overrides are hard. Please I'm just a lowly C programmer.

* fix packet size not being sent correctly

Thanks to @GlowingUmbreon on Github for troubleshooting this

* fix select(2) calls (i should read docs more properly)

* fix GDB command sequencing/parsing issue (hopefully)

* [GDB] implement no-ack mode

* fix sending ack on handshake

* get lldb to work

3 files changed, 154 insertions, 0 deletions

diff --git a/src/debug/gdb_test/.gitignore b/src/debug/gdb_test/.gitignore
new file mode 100644
index 0000000..218500b
--- /dev/null
+++ b/src/debug/gdb_test/.gitignore
@@ -0,0 +1,2 @@
+obj/
+test-gdb
diff --git a/src/debug/gdb_test/Makefile b/src/debug/gdb_test/Makefile
new file mode 100644
index 0000000..e835795
--- /dev/null
+++ b/src/debug/gdb_test/Makefile
@@ -0,0 +1,28 @@
+
+default: all
+
+all: test-gdb
+
+CPPFLAGS += -Werror=implicit-function-declaration -Werror=int-conversion \
+    -Werror=return-type -Werror=uninitialized \
+    -I../ -I../../ -Og -g -Wall \
+    -Wno-switch -Wno-pointer-sign
+
+obj/:
+	@mkdir -vp "$@"
+
+test-gdb: obj/GdbProto.o obj/GdbStub.o obj/GdbCmds.o obj/main.o obj/CRC32.o
+	$(CXX) $(CPPFLAGS) $(LDFLAGS) -o "$@" $^
+
+obj/Gdb%.o: ../Gdb%.cpp obj/
+	$(CXX) $(CPPFLAGS) -c -o "$@" "$<"
+
+obj/main.o: main.cpp obj/
+	$(CXX) $(CPPFLAGS) -c -o "$@" "$<"
+
+obj/CRC32.o: ../../CRC32.cpp obj/
+	$(CXX) $(CPPFLAGS) -c -o "$@" "$<"
+
+clean:
+	@$(RM) -rv obj/ test-gdb
+
diff --git a/src/debug/gdb_test/main.cpp b/src/debug/gdb_test/main.cpp
new file mode 100644
index 0000000..afdfa2c
--- /dev/null
+++ b/src/debug/gdb_test/main.cpp
@@ -0,0 +1,124 @@
+
+#include <stdarg.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <time.h>
+
+#include "GdbStub.h"
+#include "Platform.h"
+
+class Debug : public Gdb::StubCallbacks
+{
+public:
+	Debug(){}
+	~Debug(){}
+
+	int GetCPU() const override { return 9; }
+
+	u32 ReadReg(Gdb::Register reg) override
+	{
+		printf("[==>] read reg %d\n", (int)reg);
+		if (reg == Gdb::Register::pc) return 0x000000df; // cpsr: irq,fiq disabled, arm, sys mode
+		else return 0x69420;
+	}
+	void WriteReg(Gdb::Register reg, u32 value) override
+	{
+		printf("[==>] write reg %d: 0x%08x\n", (int)reg, value);
+	}
+
+	u32 ReadMem(u32 addr, int len) override
+	{
+		static const u32 words[] = {
+			0xeafffffe,
+			0xe0211002,
+			0xe12fff1e,
+			0
+		};
+
+		printf("[==>] read mem 0x%08x (size %u)\n", addr, len);
+
+		// $: b $ (arm)
+		return words[(addr>>2)&3] & ((1uLL<<len)-1);
+	}
+	void WriteMem(u32 addr, int len, u32 value) override
+	{
+		printf("[==>] write addr 0x%08x (size %u): 0x%08x\n", addr, len, value);
+	}
+
+	void ResetGdb() override
+	{
+		printf("[==>] RESET!!!\n");
+	}
+	int RemoteCmd(const u8* cmd, size_t len) override
+	{
+		printf("[==>] Rcmd: %s\n", cmd);
+		return 0;
+	}
+};
+
+int main(int argc, char** argv) {
+	Debug debug;
+
+	Gdb::GdbStub stub(&debug, (argc > 1) ? atoi(argv[1]) : 3333);
+	if (!stub.Init()) return 1;
+
+	do
+	{
+		while (true)
+		{
+			Gdb::StubState s = stub.Poll();
+
+			if (s == Gdb::StubState::None || s == Gdb::StubState::NoConn)
+			{
+				struct timespec ts;
+				ts.tv_sec = 0;
+				ts.tv_nsec = 1000*1000; // 1 ms
+				nanosleep(&ts, NULL);
+				continue;
+			}
+
+			switch (s)
+			{
+			case Gdb::StubState::Attach:
+				printf("[==>] attached\n");
+				break;
+			case Gdb::StubState::Break:
+				printf("[==>] break execution\n");
+				stub.SignalStatus(Gdb::TgtStatus::BreakReq, ~(u32)0);
+				break;
+			case Gdb::StubState::Continue:
+				printf("[==>] continue execution\n");
+				// TODO: send signal status on SIGSTOP? eh.
+				break;
+			case Gdb::StubState::Step:
+				printf("[==>] single-step\n");
+				stub.SignalStatus(Gdb::TgtStatus::SingleStep, ~(u32)0);
+				break;
+			case Gdb::StubState::Disconnect:
+				printf("[==>] disconnect\n");
+				stub.SignalStatus(Gdb::TgtStatus::None, ~(u32)0);
+				break;
+			}
+
+			if (s == Gdb::StubState::Disconnect) break;
+		}
+	}
+	while (false);
+
+	stub.Close();
+	return 0;
+}
+
+namespace Platform
+{
+void Log(LogLevel level, const char* fmt, ...)
+{
+    if (fmt == nullptr) return;
+
+    va_list args;
+    va_start(args, fmt);
+    vprintf(fmt, args);
+    va_end(args);
+}
+}
+