Protect savestates while the threaded software renderer is running (#1864)

* First crack at ensuring the render thread doesn't touch GPU state while it's being serialized

* Get rid of the semaphore wait

* Add some extra fields into GPU3D's serialization

* Oops, TempVertexBuffer is already serialized

* Move vertex serialization into its own method

* Lock the GPU3D state when rendering on the render thread or serializing it

* Revert "Lock the GPU3D state when rendering on the render thread or serializing it"

This reverts commit 2f49a551c13934b9dc815bbda67a45098f0482a7.

* Add comments that describe the synchronization within GPU3D_Soft

- I need to understand it before I can solve my actual problem
- Now I do

* Revert "Revert "Lock the GPU3D state when rendering on the render thread or serializing it""

This reverts commit 1977566a6d8671d72bd94ba4ebf832c3bf08933a.

* Let's try locking the GPU3D state throughout NDS::RunFrame

- Just to see what happens

* Slim down the lock's scope

* Narrow the lock's scope some more

* Remove the lock entirely

* Try protecting the GPU3D state with just a mutex

- I'll clean this up once I know it works

* Remove a duplicate method definition

* Add a missing `noexcept` specifier

* Remove an unused function

* Cut some non-hardware state from `GPU3D`'s savestate

* Assume that the next frame after loading a savestate won't be identical

* Actually, it _is_ worth it

* Don't serialize the clip matrix

- It's recalculated anyway

* Serialize `RenderPolygonRAM` as an array of indexes

* Clean up some comments

- I liked the dialogue style, but oh well

* Try restarting the render thread instead of using the lock

- Let's see what happens

* Put the lock back

* Fix some polygon and vertex indexes being saved incorrectly

- Taking the difference between two pointers results in the number of elements, not the number of bytes

* Remove `SoftRenderer::StateBusy` since it turns out we don't need it

- The real synchronization was the friends we made along the way

1 files changed, 65 insertions, 42 deletions

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index 4787702..47abae2 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -146,6 +146,19 @@ GPU3D::GPU3D(melonDS::NDS& nds, std::unique_ptr<Renderer3D>&& renderer) noexcept
 {
 }
 
+void Vertex::DoSavestate(Savestate* file) noexcept
+{
+    file->VarArray(Position, sizeof(Position));
+    file->VarArray(Color, sizeof(Color));
+    file->VarArray(TexCoords, sizeof(TexCoords));
+
+    file->Bool32(&Clipped);
+
+    file->VarArray(FinalPosition, sizeof(FinalPosition));
+    file->VarArray(FinalColor, sizeof(FinalColor));
+    file->VarArray(HiresPosition, sizeof(HiresPosition));
+}
+
 void GPU3D::SetCurrentRenderer(std::unique_ptr<Renderer3D>&& renderer) noexcept
 {
     CurrentRenderer = std::move(renderer);
@@ -297,6 +310,12 @@ void GPU3D::DoSavestate(Savestate* file) noexcept
 {
     file->Section("GP3D");
 
+    SoftRenderer* softRenderer = dynamic_cast<SoftRenderer*>(CurrentRenderer.get());
+    if (softRenderer && softRenderer->IsThreaded())
+    {
+        softRenderer->SetupRenderThread(NDS.GPU);
+    }
+
     CmdFIFO.DoSavestate(file);
     CmdPIPE.DoSavestate(file);
 
@@ -372,33 +391,21 @@ void GPU3D::DoSavestate(Savestate* file) noexcept
     file->Var32(&VertexNumInPoly);
     file->Var32(&NumConsecutivePolygons);
 
-    for (int i = 0; i < 4; i++)
+    for (Vertex& vtx : TempVertexBuffer)
     {
-        Vertex* vtx = &TempVertexBuffer[i];
-
-        file->VarArray(vtx->Position, sizeof(s32)*4);
-        file->VarArray(vtx->Color, sizeof(s32)*3);
-        file->VarArray(vtx->TexCoords, sizeof(s16)*2);
-
-        file->Bool32(&vtx->Clipped);
-
-        file->VarArray(vtx->FinalPosition, sizeof(s32)*2);
-        file->VarArray(vtx->FinalColor, sizeof(s32)*3);
+        vtx.DoSavestate(file);
     }
 
     if (file->Saving)
     {
-        u32 id;
-        if (LastStripPolygon) id = (u32)((LastStripPolygon - (&PolygonRAM[0])) / sizeof(Polygon));
-        else                  id = -1;
-        file->Var32(&id);
+        u32 index = LastStripPolygon ? (u32)(LastStripPolygon - &PolygonRAM[0]) : UINT32_MAX;
+        file->Var32(&index);
     }
     else
     {
-        u32 id;
-        file->Var32(&id);
-        if (id == 0xFFFFFFFF) LastStripPolygon = NULL;
-        else          LastStripPolygon = &PolygonRAM[id];
+        u32 index = UINT32_MAX;
+        file->Var32(&index);
+        LastStripPolygon = (index == UINT32_MAX) ? nullptr : &PolygonRAM[index];
     }
 
     file->Var32(&CurRAMBank);
@@ -409,18 +416,9 @@ void GPU3D::DoSavestate(Savestate* file) noexcept
     file->Var32(&FlushRequest);
     file->Var32(&FlushAttributes);
 
-    for (int i = 0; i < 6144*2; i++)
+    for (Vertex& vtx : VertexRAM)
     {
-        Vertex* vtx = &VertexRAM[i];
-
-        file->VarArray(vtx->Position, sizeof(s32)*4);
-        file->VarArray(vtx->Color, sizeof(s32)*3);
-        file->VarArray(vtx->TexCoords, sizeof(s16)*2);
-
-        file->Bool32(&vtx->Clipped);
-
-        file->VarArray(vtx->FinalPosition, sizeof(s32)*2);
-        file->VarArray(vtx->FinalColor, sizeof(s32)*3);
+        vtx.DoSavestate(file);
     }
 
     for(int i = 0; i < 2048*2; i++)
@@ -434,20 +432,17 @@ void GPU3D::DoSavestate(Savestate* file) noexcept
             for (int j = 0; j < 10; j++)
             {
                 Vertex* ptr = poly->Vertices[j];
-                u32 id;
-                if (ptr) id = (u32)((ptr - (&VertexRAM[0])) / sizeof(Vertex));
-                else     id = -1;
-                file->Var32(&id);
+                u32 index = ptr ? (u32)(ptr - &VertexRAM[0]) : UINT32_MAX;
+                file->Var32(&index);
             }
         }
         else
         {
             for (int j = 0; j < 10; j++)
             {
-                u32 id = -1;
-                file->Var32(&id);
-                if (id == 0xFFFFFFFF) poly->Vertices[j] = NULL;
-                else          poly->Vertices[j] = &VertexRAM[id];
+                u32 index = UINT32_MAX;
+                file->Var32(&index);
+                poly->Vertices[j] = index == UINT32_MAX ? nullptr : &VertexRAM[index];
             }
         }
 
@@ -495,7 +490,6 @@ void GPU3D::DoSavestate(Savestate* file) noexcept
         }
     }
 
-    // probably not worth storing the vblank-latched Renderxxxxxx variables
     CmdStallQueue.DoSavestate(file);
 
     file->Var32((u32*)&VertexPipeline);
@@ -511,10 +505,27 @@ void GPU3D::DoSavestate(Savestate* file) noexcept
 
         CurVertexRAM = &VertexRAM[CurRAMBank ? 6144 : 0];
         CurPolygonRAM = &PolygonRAM[CurRAMBank ? 2048 : 0];
+    }
+
+    file->Var32(&RenderNumPolygons);
+    if (file->Saving)
+    {
+        for (const Polygon* p : RenderPolygonRAM)
+        {
+            u32 index = p ? (p - &PolygonRAM[0]) : UINT32_MAX;
+
+            file->Var32(&index);
+        }
+    }
+    else
+    {
+        for (int i = 0; i < RenderPolygonRAM.size(); ++i)
+        {
+            u32 index = UINT32_MAX;
+            file->Var32(&index);
 
-        // better safe than sorry, I guess
-        // might cause a blank frame but atleast it won't shit itself
-        RenderNumPolygons = 0;
+            RenderPolygonRAM[i] = index == UINT32_MAX ? nullptr : &PolygonRAM[index];
+        }
     }
 
     file->VarArray(CurVertex, sizeof(s16)*3);
@@ -534,6 +545,18 @@ void GPU3D::DoSavestate(Savestate* file) noexcept
     file->VarArray(ShininessTable, 128*sizeof(u8));
 
     file->Bool32(&AbortFrame);
+    file->Bool32(&GeometryEnabled);
+    file->Bool32(&RenderingEnabled);
+    file->Var32(&PolygonMode);
+    file->Var32(&PolygonAttr);
+    file->Var32(&CurPolygonAttr);
+    file->Var32(&TexParam);
+    file->Var32(&TexPalette);
+    RenderFrameIdentical = false;
+    if (softRenderer && softRenderer->IsThreaded())
+    {
+        softRenderer->EnableRenderThread();
+    }
 }