From 736bd73b28875a06627f76dc2703127ccb9151b9 Mon Sep 17 00:00:00 2001 From: Quajak Date: Thu, 12 Oct 2023 21:48:12 -0400 Subject: [PATCH 1/2] Allow enabling of automatic gc trigger when number of free pages gets too low --- Docs/articles/Kernel/MemoryManagement.md | 4 + Test.sln | 79 -------------------- Tests/Cosmos.Core.Memory.Test/RATTest.cs | 6 ++ Tests/Kernels/MemoryOperationsTest/Kernel.cs | 26 +++++++ source/Cosmos.Core/Memory/RAT.cs | 36 ++++++++- 5 files changed, 70 insertions(+), 81 deletions(-) diff --git a/Docs/articles/Kernel/MemoryManagement.md b/Docs/articles/Kernel/MemoryManagement.md index 02de485985..795b05e9f4 100644 --- a/Docs/articles/Kernel/MemoryManagement.md +++ b/Docs/articles/Kernel/MemoryManagement.md @@ -45,6 +45,10 @@ The garbage collector has to be manually triggerd using the call `int Heap.Colle Note that the GC does not track objects only pointed to by pointers. To ensure that the GC nevertheless does not incorrectly free objects, you can use `void GCImplementation.IncRootCount(ushort* aPtr)` to manually increase the references of your object by 1. Once you no longer need the object you can use `void GCImplementation.DecRootCount(ushort* aPtr)` to remove the manual reference, which allows the next `Heap.Collect` call to free the object. +## Automatically Trigger Garbage Collection + +When `RAT.MinFreePages` is set to a positive value and the number of free pages (as tracked by `RAT.FreePageCount`) drops below this value, on page allocation `Heap.Collect` will automatically be called. Each time this happens the value in `RAT.GCTriggered` is incremented by one. + ### Internals The garbage collector uses the tracing approach, which means that during collection a graph of all reachable objects is created and all non-discovered objects are freed. The garbage collector will only check objects on pages which have a type where the `GCManaged` bit is set. The graph is created by starting from "root" objects which are either stored in static variables or part of the current stack. Each of these objects is "marked" and all objects referenced by this object are recursivly also "marked" and "swept". This is done using the methods `void Heap.MarkAndSweepObject(void* aPtr)` for objects and `void Heap.SweepTypedObject(uint* obj, uint type)` for structures. For this to work each allocated object holds a 1bit flag if the object was discovered during the marking phase and a 7bit value counter for the number of static references it has. The number of static references an object has is updated using `void GCImplementation.IncRootCount(ushort* aPtr)` and similar methods, which are called from the Stsfld opcode. diff --git a/Test.sln b/Test.sln index ea37746414..ecd1cbf225 100644 --- a/Test.sln +++ b/Test.sln @@ -106,38 +106,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.Core.Memory.Test", " EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.Core_Asm", "source\Cosmos.Core_Asm\Cosmos.Core_Asm.csproj", "{B7077A34-D7F0-4422-BE7C-65DF26C65489}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Kernel G3", "Kernel G3", "{99192440-2DD7-4E71-B730-D44A73F46533}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "10 CPU", "10 CPU", "{29B893F7-6C0F-4710-A60E-7FB3498BCA63}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "20 Platform", "20 Platform", "{B4CB7BF5-CADF-4056-9C09-EAAC50BC76C0}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "30 HAL", "30 HAL", "{E4299234-8323-43F6-B684-350A1232746B}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "40 System", "40 System", "{DE38917F-969B-486C-AF83-C59E5E52400A}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "50 Application", "50 Application", "{02FF94AF-6BA3-49ED-A027-A63F591C310D}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "91 Plug", "91 Plug", "{1FC213DE-5033-40E1-9C16-5F1A0CDC9693}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.CPU.x86", "source\Kernel-X86\10-CPU\Cosmos.CPU.x86\Cosmos.CPU.x86.csproj", "{4D219A6D-4528-4622-AF29-96F830C4D076}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.Platform.PC", "source\Kernel-X86\20-Platform\Cosmos.Platform.PC\Cosmos.Platform.PC.csproj", "{6CBABA8D-4207-4E1E-8122-63DB51D25F18}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.HAL", "source\Kernel-X86\30-HAL\Cosmos.HAL\Cosmos.HAL.csproj", "{16ECD6DE-6F65-4A5C-8B49-A29782D9D057}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.System", "source\Kernel-X86\40-System\Cosmos.System\Cosmos.System.csproj", "{F588033A-6B7D-4ABF-96C4-73D8B2271A6B}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.Plugs.TapRoot", "source\Kernel-X86\91-Plugs\Cosmos.Plugs.TapRoot\Cosmos.Plugs.TapRoot.csproj", "{756ECECD-B213-42F0-BF58-4A91B4C47FAA}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.CPU_Plugs", "source\Kernel-X86\10-CPU\Cosmos.CPU_Plugs\Cosmos.CPU_Plugs.csproj", "{C000BFB2-DFDE-4B1E-BDA6-988B30370C7A}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.CPU_Asm", "source\Kernel-X86\10-CPU\Cosmos.CPU_Asm\Cosmos.CPU_Asm.csproj", "{0C7C9F9D-6498-45E8-B77B-FF4D381C3297}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TheRingMaster", "source\TheRingMaster\TheRingMaster.csproj", "{3DD192AF-2D72-449F-936C-ED8734225B18}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "92 CpuPlug", "92 CpuPlug", "{929EE8ED-6AD3-4442-A0C1-EC70665F2DCF}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spruce", "..\XSharp\source\Spruce\Spruce.csproj", "{FF46829E-B612-4D36-80BE-ED04521AD91A}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.Compiler.Tests.TypeSystem", "Tests\Kernels\Cosmos.Compiler.Tests.TypeSystem\Cosmos.Compiler.Tests.TypeSystem.csproj", "{D21A7C6C-A696-4EC3-84EB-70700C1E3B34}" @@ -657,26 +625,6 @@ Global {F588033A-6B7D-4ABF-96C4-73D8B2271A6B}.Release|x86.Build.0 = Release|Any CPU {F588033A-6B7D-4ABF-96C4-73D8B2271A6B}.TEST|Any CPU.ActiveCfg = Release|Any CPU {F588033A-6B7D-4ABF-96C4-73D8B2271A6B}.TEST|x86.ActiveCfg = Release|Any CPU - {756ECECD-B213-42F0-BF58-4A91B4C47FAA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {756ECECD-B213-42F0-BF58-4A91B4C47FAA}.Debug|Any CPU.Build.0 = Debug|Any CPU - {756ECECD-B213-42F0-BF58-4A91B4C47FAA}.Debug|x86.ActiveCfg = Debug|Any CPU - {756ECECD-B213-42F0-BF58-4A91B4C47FAA}.Debug|x86.Build.0 = Debug|Any CPU - {756ECECD-B213-42F0-BF58-4A91B4C47FAA}.Release|Any CPU.ActiveCfg = Release|Any CPU - {756ECECD-B213-42F0-BF58-4A91B4C47FAA}.Release|Any CPU.Build.0 = Release|Any CPU - {756ECECD-B213-42F0-BF58-4A91B4C47FAA}.Release|x86.ActiveCfg = Release|Any CPU - {756ECECD-B213-42F0-BF58-4A91B4C47FAA}.Release|x86.Build.0 = Release|Any CPU - {756ECECD-B213-42F0-BF58-4A91B4C47FAA}.TEST|Any CPU.ActiveCfg = Release|Any CPU - {756ECECD-B213-42F0-BF58-4A91B4C47FAA}.TEST|x86.ActiveCfg = Release|Any CPU - {C000BFB2-DFDE-4B1E-BDA6-988B30370C7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C000BFB2-DFDE-4B1E-BDA6-988B30370C7A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C000BFB2-DFDE-4B1E-BDA6-988B30370C7A}.Debug|x86.ActiveCfg = Debug|Any CPU - {C000BFB2-DFDE-4B1E-BDA6-988B30370C7A}.Debug|x86.Build.0 = Debug|Any CPU - {C000BFB2-DFDE-4B1E-BDA6-988B30370C7A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C000BFB2-DFDE-4B1E-BDA6-988B30370C7A}.Release|Any CPU.Build.0 = Release|Any CPU - {C000BFB2-DFDE-4B1E-BDA6-988B30370C7A}.Release|x86.ActiveCfg = Release|Any CPU - {C000BFB2-DFDE-4B1E-BDA6-988B30370C7A}.Release|x86.Build.0 = Release|Any CPU - {C000BFB2-DFDE-4B1E-BDA6-988B30370C7A}.TEST|Any CPU.ActiveCfg = Release|Any CPU - {C000BFB2-DFDE-4B1E-BDA6-988B30370C7A}.TEST|x86.ActiveCfg = Release|Any CPU {0C7C9F9D-6498-45E8-B77B-FF4D381C3297}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {0C7C9F9D-6498-45E8-B77B-FF4D381C3297}.Debug|Any CPU.Build.0 = Debug|Any CPU {0C7C9F9D-6498-45E8-B77B-FF4D381C3297}.Debug|x86.ActiveCfg = Debug|Any CPU @@ -687,18 +635,6 @@ Global {0C7C9F9D-6498-45E8-B77B-FF4D381C3297}.Release|x86.Build.0 = Release|Any CPU {0C7C9F9D-6498-45E8-B77B-FF4D381C3297}.TEST|Any CPU.ActiveCfg = Release|Any CPU {0C7C9F9D-6498-45E8-B77B-FF4D381C3297}.TEST|x86.ActiveCfg = Release|Any CPU - {3DD192AF-2D72-449F-936C-ED8734225B18}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3DD192AF-2D72-449F-936C-ED8734225B18}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3DD192AF-2D72-449F-936C-ED8734225B18}.Debug|x86.ActiveCfg = Debug|Any CPU - {3DD192AF-2D72-449F-936C-ED8734225B18}.Debug|x86.Build.0 = Debug|Any CPU - {3DD192AF-2D72-449F-936C-ED8734225B18}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3DD192AF-2D72-449F-936C-ED8734225B18}.Release|Any CPU.Build.0 = Release|Any CPU - {3DD192AF-2D72-449F-936C-ED8734225B18}.Release|x86.ActiveCfg = Release|Any CPU - {3DD192AF-2D72-449F-936C-ED8734225B18}.Release|x86.Build.0 = Release|Any CPU - {3DD192AF-2D72-449F-936C-ED8734225B18}.TEST|Any CPU.ActiveCfg = TEST|Any CPU - {3DD192AF-2D72-449F-936C-ED8734225B18}.TEST|Any CPU.Build.0 = TEST|Any CPU - {3DD192AF-2D72-449F-936C-ED8734225B18}.TEST|x86.ActiveCfg = TEST|Any CPU - {3DD192AF-2D72-449F-936C-ED8734225B18}.TEST|x86.Build.0 = TEST|Any CPU {FF46829E-B612-4D36-80BE-ED04521AD91A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {FF46829E-B612-4D36-80BE-ED04521AD91A}.Debug|Any CPU.Build.0 = Debug|Any CPU {FF46829E-B612-4D36-80BE-ED04521AD91A}.Debug|x86.ActiveCfg = Debug|Any CPU @@ -953,21 +889,6 @@ Global {FB23BD72-AEC3-485E-B86C-8E7DB0B3BB9B} = {29EEC029-6A2B-478A-B6E5-D63A91388ABA} {408E5ACC-EA9A-41E8-AA95-514C5F47BD34} = {52D81759-C7CC-427F-8C96-89CA10C914B5} {B7077A34-D7F0-4422-BE7C-65DF26C65489} = {04B18FFC-8EA0-4E9F-9E1B-478527B19AFA} - {29B893F7-6C0F-4710-A60E-7FB3498BCA63} = {99192440-2DD7-4E71-B730-D44A73F46533} - {B4CB7BF5-CADF-4056-9C09-EAAC50BC76C0} = {99192440-2DD7-4E71-B730-D44A73F46533} - {E4299234-8323-43F6-B684-350A1232746B} = {99192440-2DD7-4E71-B730-D44A73F46533} - {DE38917F-969B-486C-AF83-C59E5E52400A} = {99192440-2DD7-4E71-B730-D44A73F46533} - {02FF94AF-6BA3-49ED-A027-A63F591C310D} = {99192440-2DD7-4E71-B730-D44A73F46533} - {1FC213DE-5033-40E1-9C16-5F1A0CDC9693} = {99192440-2DD7-4E71-B730-D44A73F46533} - {4D219A6D-4528-4622-AF29-96F830C4D076} = {29B893F7-6C0F-4710-A60E-7FB3498BCA63} - {6CBABA8D-4207-4E1E-8122-63DB51D25F18} = {B4CB7BF5-CADF-4056-9C09-EAAC50BC76C0} - {16ECD6DE-6F65-4A5C-8B49-A29782D9D057} = {E4299234-8323-43F6-B684-350A1232746B} - {F588033A-6B7D-4ABF-96C4-73D8B2271A6B} = {DE38917F-969B-486C-AF83-C59E5E52400A} - {756ECECD-B213-42F0-BF58-4A91B4C47FAA} = {1FC213DE-5033-40E1-9C16-5F1A0CDC9693} - {C000BFB2-DFDE-4B1E-BDA6-988B30370C7A} = {929EE8ED-6AD3-4442-A0C1-EC70665F2DCF} - {0C7C9F9D-6498-45E8-B77B-FF4D381C3297} = {29B893F7-6C0F-4710-A60E-7FB3498BCA63} - {3DD192AF-2D72-449F-936C-ED8734225B18} = {C286932C-3F6D-47F0-BEEF-26843D1BB11B} - {929EE8ED-6AD3-4442-A0C1-EC70665F2DCF} = {99192440-2DD7-4E71-B730-D44A73F46533} {FF46829E-B612-4D36-80BE-ED04521AD91A} = {E9CD521E-C386-466D-B5F7-A5EB19A61625} {D21A7C6C-A696-4EC3-84EB-70700C1E3B34} = {ECEA7778-E786-4317-90B9-A2D4427CB91C} {0DF97CAC-220B-4DAD-B397-42E394255763} = {ECEA7778-E786-4317-90B9-A2D4427CB91C} diff --git a/Tests/Cosmos.Core.Memory.Test/RATTest.cs b/Tests/Cosmos.Core.Memory.Test/RATTest.cs index 3366258c30..8fae5b4a77 100644 --- a/Tests/Cosmos.Core.Memory.Test/RATTest.cs +++ b/Tests/Cosmos.Core.Memory.Test/RATTest.cs @@ -44,10 +44,16 @@ public unsafe void RATMethods() RAT.Init(xPtr, (uint)xRAM.Length); + uint freePageCount = RAT.FreePageCount; + Assert.IsTrue(freePageCount < RAT.TotalPageCount); + Assert.AreEqual(freePageCount, RAT.GetPageCount((byte)RAT.PageType.Empty)); + var largePage = RAT.AllocPages(RAT.PageType.HeapLarge, 3); Assert.AreEqual(RAT.PageType.HeapLarge, RAT.GetPageType(largePage)); Assert.AreEqual(RAT.GetFirstRATIndex(largePage), RAT.GetFirstRATIndex((byte*)largePage + 20)); Assert.AreEqual(RAT.PageType.HeapLarge, RAT.GetPageType((byte*)largePage + RAT.PageSize)); + + Assert.AreEqual(RAT.FreePageCount, freePageCount - 3); } } diff --git a/Tests/Kernels/MemoryOperationsTest/Kernel.cs b/Tests/Kernels/MemoryOperationsTest/Kernel.cs index d3cef0968e..51ae35148c 100644 --- a/Tests/Kernels/MemoryOperationsTest/Kernel.cs +++ b/Tests/Kernels/MemoryOperationsTest/Kernel.cs @@ -157,6 +157,7 @@ static unsafe void TestMemoryBlock(MemoryBlock memoryBlock) memoryBlock.Read32(read); Assert.AreEqual(values, read, "Using Fill(int, int, int) works"); } + static unsafe void TestRealloc() { // Allocate initial pointer and fill with value 32 @@ -174,15 +175,40 @@ static unsafe void TestRealloc() Assert.AreEqual(aPtr[16], 0, "Expected value 0 not found at the end of aPtr."); } + static unsafe void TestMemoryManager() + { + uint freePageCount = RAT.FreePageCount; + Assert.IsTrue(freePageCount < RAT.TotalPageCount, "Number of free pages is less than total number of pages"); + Assert.AreEqual(freePageCount, RAT.GetPageCount((byte)RAT.PageType.Empty), "GetPageCount and FreePageCount get different number of free pages"); + void* pointer = RAT.AllocPages(RAT.PageType.HeapLarge, 3); + Assert.AreEqual(RAT.FreePageCount, freePageCount - 3, "Allocating three pages reduces number of free pages by three"); + RAT.Free(RAT.GetFirstRATIndex(pointer)); + Assert.AreEqual(RAT.FreePageCount, freePageCount, "Freeing the allocated pages correctly updates the number of free pages"); + } + + void TestAutomaticGCCollect() + { + Assert.AreEqual(RAT.GCTriggered, 0, "Before Enabling GC hasnt been triggered automatically"); + RAT.MinFreePages = (int)RAT.FreePageCount - 10; + while (RAT.GCTriggered == 0) + { + Console.WriteLine($"Free: {RAT.FreePageCount} Min: {RAT.MinFreePages} GC Triggered: {RAT.GCTriggered} Objects: {HeapSmall.GetAllocatedObjectCount()}"); + } + Assert.Succeed("GC can be triggered automatically"); + RAT.MinFreePages = -1; + } + protected override void Run() { try { + TestMemoryManager(); TestCopy(); TestMemoryBlock(new MemoryBlock(0x60000, 128)); //we are testing in SVGA video memory which should not be in use TestManagedMemoryBlock(new ManagedMemoryBlock(128)); TestRealloc(); SpanTest.Execute(); + TestAutomaticGCCollect(); TestController.Completed(); } catch (Exception e) diff --git a/source/Cosmos.Core/Memory/RAT.cs b/source/Cosmos.Core/Memory/RAT.cs index b62c8c3a8f..0bcb878a7d 100644 --- a/source/Cosmos.Core/Memory/RAT.cs +++ b/source/Cosmos.Core/Memory/RAT.cs @@ -21,7 +21,7 @@ unsafe static public class RAT /// /// PageType enum. Used to define the type of the page. /// - public enum PageType + public enum PageType : byte { /// /// Empty page. @@ -93,7 +93,23 @@ public enum PageType /// Number of pages in the heap. /// /// Calculated from mSize. - static public uint TotalPageCount; + public static uint TotalPageCount; + + /// + /// Number of pages which are currently not in use + /// + public static uint FreePageCount; + + /// + /// If number of free pages drops below this number, we trigger the GC.Collect automatically + /// If set to -1 it is disabled + /// + public static int MinFreePages = -1; + + /// + /// Number of the times the GC has been triggered automatically + /// + public static uint GCTriggered = 0; /// /// Pointer to the RAT. @@ -132,6 +148,7 @@ public static void Init(byte* aStartPtr, uint aSize) RamSize = aSize; HeapEnd = aStartPtr + aSize; TotalPageCount = aSize / PageSize; + FreePageCount = TotalPageCount; // We need one status byte for each block. // Intel blocks are 4k (10 bits). So for 4GB, this means @@ -150,6 +167,9 @@ public static void Init(byte* aStartPtr, uint aSize) { *p = (byte)PageType.RAT; } + // Remove pages needed for RAT table from count + FreePageCount -= xRatPageCount; + Heap.Init(); } @@ -192,6 +212,12 @@ public static uint GetPageCount(byte aType = 0) /// A pointer to the first page on success, null on failure. public static void* AllocPages(PageType aType, uint aPageCount = 1) { + if (MinFreePages > 0 && MinFreePages > FreePageCount) + { + Heap.Collect(); + GCTriggered++; + } + byte* xPos = null; // Could combine with an external method or delegate, but will slow things down @@ -241,6 +267,10 @@ public static uint GetPageCount(byte aType = 0) *p = (byte)PageType.Extension; } CPU.ZeroFill((uint)xResult, PageSize * aPageCount); + + // Decrement free page count + FreePageCount -= aPageCount; + return xResult; } return null; @@ -295,6 +325,7 @@ public static void Free(uint aPageIdx) { byte* p = mRAT + aPageIdx; *p = (byte)PageType.Empty; + FreePageCount++; for (; p < mRAT + TotalPageCount; ) { if (*++p != (byte)PageType.Extension) @@ -302,6 +333,7 @@ public static void Free(uint aPageIdx) break; } *p = (byte)PageType.Empty; + FreePageCount++; } } } From fcc8c63eb4194b96181fd57106c14b6a9a50fd9f Mon Sep 17 00:00:00 2001 From: Quajak Date: Thu, 12 Oct 2023 21:48:28 -0400 Subject: [PATCH 2/2] Simplify check --- source/Cosmos.Core/Memory/RAT.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Cosmos.Core/Memory/RAT.cs b/source/Cosmos.Core/Memory/RAT.cs index 0bcb878a7d..2ff6a9c418 100644 --- a/source/Cosmos.Core/Memory/RAT.cs +++ b/source/Cosmos.Core/Memory/RAT.cs @@ -212,7 +212,7 @@ public static uint GetPageCount(byte aType = 0) /// A pointer to the first page on success, null on failure. public static void* AllocPages(PageType aType, uint aPageCount = 1) { - if (MinFreePages > 0 && MinFreePages > FreePageCount) + if (MinFreePages > FreePageCount) { Heap.Collect(); GCTriggered++;