From c4fc94eab48a8f3bd6ecb8919b1a50a756068034 Mon Sep 17 00:00:00 2001 From: Mukund Raghav Sharma Date: Wed, 8 Jan 2025 11:55:46 -0800 Subject: [PATCH 1/6] OOM diag work --- src/PerfView/GcStats.cs | 1 + .../Computers/TraceManagedProcess.cs | 18 +++ .../Parsers/GCDynamicTraceEventParser.cs | 132 +++++++++++++++++- src/TraceEvent/TraceEvent.cs | 4 +- 4 files changed, 153 insertions(+), 2 deletions(-) diff --git a/src/PerfView/GcStats.cs b/src/PerfView/GcStats.cs index e0e8b099c..91640b17c 100644 --- a/src/PerfView/GcStats.cs +++ b/src/PerfView/GcStats.cs @@ -517,6 +517,7 @@ public static void ToXmlAttribs(TextWriter writer, TraceProcess stats, TraceLoad if (gc.TimingInfo[(int)TraceGC.TimingType.MarkScanFinalization].HasValue) { writer.Write(" MarkScanFinalization=\"{0}\"", gc.TimingInfo[(int)TraceGC.TimingType.MarkScanFinalization].Value); } if (gc.TimingInfo[(int)TraceGC.TimingType.MarkShortWeak].HasValue) { writer.Write(" MarkShortWeak=\"{0}\"", gc.TimingInfo[(int)TraceGC.TimingType.MarkShortWeak].Value); } if (gc.TimingInfo[(int)TraceGC.TimingType.MarkLongWeak].HasValue) { writer.Write(" MarkLongWeak=\"{0}\"", gc.TimingInfo[(int)TraceGC.TimingType.MarkLongWeak].Value); } + if (gc.TimingInfo[(int)TraceGC.TimingType.MarkShortWeak].HasValue) { writer.Write(" MarkShortWeak=\"{0}\"", gc.TimingInfo[(int)TraceGC.TimingType.MarkShortWeak].Value); } if (gc.TimingInfo[(int)TraceGC.TimingType.Plan].HasValue) { writer.Write(" Plan=\"{0}\"", gc.TimingInfo[(int)TraceGC.TimingType.Plan].Value); } if (gc.TimingInfo[(int)TraceGC.TimingType.Relocate].HasValue) { writer.Write(" Relocate=\"{0}\"", gc.TimingInfo[(int)TraceGC.TimingType.Relocate].Value); } if (gc.TimingInfo[(int)TraceGC.TimingType.Compact].HasValue) { writer.Write(" Compact=\"{0}\"", gc.TimingInfo[(int)TraceGC.TimingType.Compact].Value); } diff --git a/src/TraceEvent/Computers/TraceManagedProcess.cs b/src/TraceEvent/Computers/TraceManagedProcess.cs index 361c27a02..ed01944d0 100644 --- a/src/TraceEvent/Computers/TraceManagedProcess.cs +++ b/src/TraceEvent/Computers/TraceManagedProcess.cs @@ -879,6 +879,12 @@ internal static void SetupCallbacks(TraceEventDispatcher source) GCStats.ProcessCommittedUsage(stats, committedUsage); }; + source.Clr.GCDynamicEvent.GCOOMDetails += delegate (OOMDetailsTraceEvent oomDetails) + { + var stats = currentManagedProcess(oomDetails.UnderlyingEvent); + GCStats.ProcessOOMEvent(stats, oomDetails); + }; + source.Clr.GCDynamicEvent.GCDynamicTraceEvent += delegate (GCDynamicTraceEvent gcDynamic) { var stats = currentManagedProcess(gcDynamic.UnderlyingEvent); @@ -2146,6 +2152,7 @@ public double PromotedMB public CommittedUsage CommittedUsageBefore { get; internal set; } public CommittedUsage CommittedUsageAfter { get; internal set; } + public List OOMDetails { get; internal set; } = new List(); /// /// Memory survival percentage by generation @@ -4994,6 +5001,17 @@ internal static void ProcessCommittedUsage(TraceLoadedDotNetRuntime proc, Commit } } + internal static void ProcessOOMEvent(TraceLoadedDotNetRuntime proc, OOMDetailsTraceEvent oomDetailsTrace) + { + TraceGC _event = GetLastGC(proc); + _event.OOMDetails.Add(new OOMDetails + { + GCIndex = oomDetailsTrace.GCIndex, + Allocated = oomDetailsTrace.Allocated, + Size = oomDetailsTrace.Size, + }); + } + internal static void ProcessGCDynamicEvent(TraceLoadedDotNetRuntime proc, GCDynamicTraceEvent gcDynamic) { TraceGC _event = GetLastGC(proc); diff --git a/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs b/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs index 43a267cc8..e4431a64a 100644 --- a/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs +++ b/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs @@ -32,6 +32,7 @@ public GCDynamicTraceEventParser(TraceEventSource source) : base(source) // They ensure that Dispatch is called so that the specific event handlers are called for each event. ((ITraceParserServices)source).RegisterEventTemplate(GCDynamicTemplate(Dispatch, GCDynamicEventBase.GCDynamicTemplate)); ((ITraceParserServices)source).RegisterEventTemplate(GCDynamicTemplate(Dispatch, GCDynamicEventBase.CommittedUsageTemplate)); + ((ITraceParserServices)source).RegisterEventTemplate(GCDynamicTemplate(Dispatch, GCDynamicEventBase.OOMDetailsTemplate)); } protected override string GetProviderName() @@ -43,7 +44,7 @@ protected internal override void EnumerateTemplates(Func GCCommittedUsage } } + private event Action _OOMDetails; + public event Action GCOOMDetails + { + add + { + _OOMDetails += value; + } + remove + { + _OOMDetails -= value; + } + } + /// /// Responsible for dispatching the event after we determine its type /// and parse it. @@ -98,6 +113,12 @@ private void Dispatch(GCDynamicTraceEventImpl data) _gcCommittedUsage(data.EventPayload as CommittedUsageTraceEvent); } + else if (_OOMDetails != null && + data.eventID == GCDynamicEventBase.OOMDetailsTemplate.ID) + { + _OOMDetails(data.EventPayload as OOMDetailsTraceEvent); + } + else if (_gcDynamicTraceEvent != null && data.EventPayload is GCDynamicTraceEvent) { @@ -160,6 +181,7 @@ protected internal override Delegate Target private readonly CommittedUsageTraceEvent _committedUsageTemplate = new CommittedUsageTraceEvent(); private readonly GCDynamicTraceEvent _gcDynamicTemplate = new GCDynamicTraceEvent(); + private readonly OOMDetailsTraceEvent _oomDetailsTemplate = new OOMDetailsTraceEvent(); /// /// Contains the fully parsed payload of the dynamic event. @@ -173,6 +195,11 @@ public GCDynamicEventBase EventPayload return _committedUsageTemplate.Bind(this); } + else if (eventID == GCDynamicEventBase.OOMDetailsTemplate.ID) + { + return _oomDetailsTemplate.Bind(this); + } + return _gcDynamicTemplate.Bind(this); } } @@ -214,6 +241,11 @@ private void SelectEventMetadata() eventTemplate = GCDynamicEventBase.CommittedUsageTemplate; } + else if (Name.Equals(GCDynamicEventBase.OOMDetailsTemplate.OpcodeName, StringComparison.InvariantCultureIgnoreCase)) + { + eventTemplate = GCDynamicEventBase.OOMDetailsTemplate; + } + SetMetadataFromTemplate(eventTemplate); } @@ -237,6 +269,7 @@ public abstract class GCDynamicEventBase /// internal static readonly GCDynamicTraceEvent GCDynamicTemplate = new GCDynamicTraceEvent(); internal static readonly CommittedUsageTraceEvent CommittedUsageTemplate = new CommittedUsageTraceEvent(); + internal static readonly OOMDetailsTraceEvent OOMDetailsTemplate = new OOMDetailsTraceEvent(); /// /// Metadata that must be specified for each specific type of dynamic event. @@ -329,6 +362,103 @@ internal override IEnumerable> PayloadValues } } + public sealed class OOMDetailsTraceEvent : GCDynamicEventBase + { + public short Version { get { return BitConverter.ToInt16(DataField, 0); } } + public ulong GCIndex { get { return BitConverter.ToUInt64(DataField, 2); } } + public ushort Allocated { get { return BitConverter.ToUInt16(DataField, 10); } } + public ushort Reserved { get { return BitConverter.ToUInt16(DataField, 12); } } + public ulong AllocSize { get { return BitConverter.ToUInt64(DataField, 14); } } + public ushort Reason { get { return BitConverter.ToUInt16(DataField, 22); } } + public ushort FailToGetMemory { get { return BitConverter.ToUInt16(DataField, 24); } } + public ulong Size { get { return BitConverter.ToUInt16(DataField, 26); } } + public ushort IsLOH { get { return BitConverter.ToUInt16(DataField, 34); } } + public uint MemoryLoad { get { return BitConverter.ToUInt16(DataField, 36); } } + + internal override TraceEventID ID => TraceEventID.Illegal - 12; + + internal override string TaskName => "GC"; + + internal override string OpcodeName => "OOMDetails"; + + internal override string EventName => "GC/OOMDetails"; + + private string[] _payloadNames; + internal override string[] PayloadNames + { + get + { + if (_payloadNames == null) + { + _payloadNames = new string[] { "Version", "GCIndex", "Allocated", "Reserved", "AllocSize", "Reason", "FailToGetMemory", "Size", "IsLOH", "MemoryLoad", "AvailablePageFileMB" }; + } + + return _payloadNames; + } + } + + internal override IEnumerable> PayloadValues + { + get + { + yield return new KeyValuePair("Version", Version); + yield return new KeyValuePair("GCIndex", GCIndex); + yield return new KeyValuePair("Allocated", Allocated); + yield return new KeyValuePair("Reserved", Reserved); + yield return new KeyValuePair("AllocSize", AllocSize); + yield return new KeyValuePair("Reason", Reason); + yield return new KeyValuePair("FailToGetMemory", FailToGetMemory); + yield return new KeyValuePair("Size", Size); + yield return new KeyValuePair("IsLOH", IsLOH); + yield return new KeyValuePair("MemoryLoad", MemoryLoad); + yield return new KeyValuePair("AvailablePageFileMB", MemoryLoad); + } + } + + internal override object PayloadValue(int index) + { + switch (index) + { + case 0: + return Version; + case 1: + return GCIndex; + case 2: + return Allocated; + case 3: + return Reserved; + case 4: + return AllocSize; + case 5: + return Reason; + case 6: + return FailToGetMemory; + case 7: + return Size; + case 8: + return IsLOH; + case 9: + return MemoryLoad; + default: + Debug.Assert(false, "Bad field index"); + return null; + } + } + } + + public sealed class OOMDetails + { + public ulong GCIndex { get; internal set; } + public ushort Allocated { get; internal set; } + public ushort Reserved { get; internal set; } + public ulong AllocSize { get; internal set; } + public ushort Reason { get; internal set; } + public ushort FailToGetMemory { get; internal set; } + public ulong Size { get; internal set; } + public ushort IsLOH { get; internal set; } + public uint MemoryLoad { get; internal set; } + } + public sealed class CommittedUsageTraceEvent : GCDynamicEventBase { public short Version { get { return BitConverter.ToInt16(DataField, 0); } } diff --git a/src/TraceEvent/TraceEvent.cs b/src/TraceEvent/TraceEvent.cs index 24dbbf480..dab6ad214 100644 --- a/src/TraceEvent/TraceEvent.cs +++ b/src/TraceEvent/TraceEvent.cs @@ -2972,6 +2972,7 @@ private void ConfirmAllEventsAreInEnumeration() if (string.Equals(GetType().Name, nameof(GCDynamicTraceEventParser), StringComparison.OrdinalIgnoreCase)) { declaredSet.Remove("CommittedUsage"); + declaredSet.Remove("OOMDetails"); } var enumSet = new SortedDictionary(); @@ -3682,7 +3683,8 @@ internal TraceEvent Lookup(TraceEventNativeMethods.EVENT_RECORD* eventRecord) // Make sure that the assert below doesn't fail by checking if _any_ of the event header ids match. bool gcDynamicTemplateEventHeaderMatch = eventRecord->EventHeader.Id == (ushort)GCDynamicEventBase.GCDynamicTemplate.ID || - eventRecord->EventHeader.Id == (ushort)GCDynamicEventBase.CommittedUsageTemplate.ID; + eventRecord->EventHeader.Id == (ushort)GCDynamicEventBase.CommittedUsageTemplate.ID || + eventRecord->EventHeader.Id == (ushort)GCDynamicEventBase.OOMDetailsTemplate.ID; // Ignore the failure for GC dynamic events because they are all // dispatched through the same template and we vary the event ID. From 18523fa0f05f5147334359c92c1af948b1ff72df Mon Sep 17 00:00:00 2001 From: Mukund Raghav Sharma Date: Tue, 14 Jan 2025 14:11:44 -0800 Subject: [PATCH 2/6] Accounted for new properties --- .../Computers/TraceManagedProcess.cs | 2 +- .../Parsers/GCDynamicTraceEventParser.cs | 56 +++++++++---------- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/src/TraceEvent/Computers/TraceManagedProcess.cs b/src/TraceEvent/Computers/TraceManagedProcess.cs index ed01944d0..c69ef8d74 100644 --- a/src/TraceEvent/Computers/TraceManagedProcess.cs +++ b/src/TraceEvent/Computers/TraceManagedProcess.cs @@ -5007,7 +5007,7 @@ internal static void ProcessOOMEvent(TraceLoadedDotNetRuntime proc, OOMDetailsTr _event.OOMDetails.Add(new OOMDetails { GCIndex = oomDetailsTrace.GCIndex, - Allocated = oomDetailsTrace.Allocated, + //Allocated = oomDetailsTrace.Allocated, Size = oomDetailsTrace.Size, }); } diff --git a/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs b/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs index e4431a64a..70cdc87e4 100644 --- a/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs +++ b/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs @@ -365,15 +365,14 @@ internal override IEnumerable> PayloadValues public sealed class OOMDetailsTraceEvent : GCDynamicEventBase { public short Version { get { return BitConverter.ToInt16(DataField, 0); } } - public ulong GCIndex { get { return BitConverter.ToUInt64(DataField, 2); } } - public ushort Allocated { get { return BitConverter.ToUInt16(DataField, 10); } } - public ushort Reserved { get { return BitConverter.ToUInt16(DataField, 12); } } - public ulong AllocSize { get { return BitConverter.ToUInt64(DataField, 14); } } - public ushort Reason { get { return BitConverter.ToUInt16(DataField, 22); } } - public ushort FailToGetMemory { get { return BitConverter.ToUInt16(DataField, 24); } } - public ulong Size { get { return BitConverter.ToUInt16(DataField, 26); } } - public ushort IsLOH { get { return BitConverter.ToUInt16(DataField, 34); } } - public uint MemoryLoad { get { return BitConverter.ToUInt16(DataField, 36); } } + public long GCIndex { get { return BitConverter.ToInt64(DataField, 2); } } + public long AllocSize { get { return BitConverter.ToInt64(DataField, 10); } } + public short Reason { get { return BitConverter.ToInt16(DataField, 18); } } + public short FailToGetMemory { get { return BitConverter.ToInt16(DataField, 20); } } + public long Size { get { return BitConverter.ToInt64(DataField, 22); } } + public short IsLOH { get { return BitConverter.ToInt16(DataField, 30); } } + public int MemoryLoad { get { return BitConverter.ToInt32(DataField, 32); } } + public long AvailablePageFileMB { get { return BitConverter.ToInt64(DataField, 36); } } internal override TraceEventID ID => TraceEventID.Illegal - 12; @@ -390,7 +389,7 @@ internal override string[] PayloadNames { if (_payloadNames == null) { - _payloadNames = new string[] { "Version", "GCIndex", "Allocated", "Reserved", "AllocSize", "Reason", "FailToGetMemory", "Size", "IsLOH", "MemoryLoad", "AvailablePageFileMB" }; + _payloadNames = new string[] { "Version", "GCIndex", "AllocSize", "Reason", "FailToGetMemory", "Size", "IsLOH", "MemoryLoad", "AvailablePageFileMB" }; } return _payloadNames; @@ -403,15 +402,13 @@ internal override IEnumerable> PayloadValues { yield return new KeyValuePair("Version", Version); yield return new KeyValuePair("GCIndex", GCIndex); - yield return new KeyValuePair("Allocated", Allocated); - yield return new KeyValuePair("Reserved", Reserved); yield return new KeyValuePair("AllocSize", AllocSize); yield return new KeyValuePair("Reason", Reason); yield return new KeyValuePair("FailToGetMemory", FailToGetMemory); yield return new KeyValuePair("Size", Size); yield return new KeyValuePair("IsLOH", IsLOH); yield return new KeyValuePair("MemoryLoad", MemoryLoad); - yield return new KeyValuePair("AvailablePageFileMB", MemoryLoad); + yield return new KeyValuePair("AvailablePageFileMB", AvailablePageFileMB); } } @@ -424,21 +421,19 @@ internal override object PayloadValue(int index) case 1: return GCIndex; case 2: - return Allocated; - case 3: - return Reserved; - case 4: return AllocSize; - case 5: + case 3: return Reason; - case 6: + case 4: return FailToGetMemory; - case 7: + case 5: return Size; - case 8: + case 6: return IsLOH; - case 9: + case 7: return MemoryLoad; + case 8: + return AvailablePageFileMB; default: Debug.Assert(false, "Bad field index"); return null; @@ -448,15 +443,14 @@ internal override object PayloadValue(int index) public sealed class OOMDetails { - public ulong GCIndex { get; internal set; } - public ushort Allocated { get; internal set; } - public ushort Reserved { get; internal set; } - public ulong AllocSize { get; internal set; } - public ushort Reason { get; internal set; } - public ushort FailToGetMemory { get; internal set; } - public ulong Size { get; internal set; } - public ushort IsLOH { get; internal set; } - public uint MemoryLoad { get; internal set; } + public long GCIndex { get; internal set; } + public long AllocSize { get; internal set; } + public short Reason { get; internal set; } + public short FailToGetMemory { get; internal set; } + public long Size { get; internal set; } + public short IsLOH { get; internal set; } + public int MemoryLoad { get; internal set; } + public long AvailablePageFileMB { get; internal set; } } public sealed class CommittedUsageTraceEvent : GCDynamicEventBase From 7ce9898618e70d6afbc9227c92fb6df1c46feb98 Mon Sep 17 00:00:00 2001 From: Mukund Raghav Sharma Date: Tue, 28 Jan 2025 14:31:12 -0800 Subject: [PATCH 3/6] Added more changes --- .../Computers/TraceManagedProcess.cs | 27 +++++++++--- .../Parsers/GCDynamicTraceEventParser.cs | 42 ++++++++++++++----- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/src/TraceEvent/Computers/TraceManagedProcess.cs b/src/TraceEvent/Computers/TraceManagedProcess.cs index c69ef8d74..d21f67f84 100644 --- a/src/TraceEvent/Computers/TraceManagedProcess.cs +++ b/src/TraceEvent/Computers/TraceManagedProcess.cs @@ -1744,6 +1744,11 @@ private void Calculate() m_stats.MaxSizePeakMB = Math.Max(m_stats.MaxSizePeakMB, _gc.HeapSizePeakMB); m_stats.MaxAllocRateMBSec = Math.Max(m_stats.MaxAllocRateMBSec, _gc.AllocRateMBSec); m_stats.MaxSuspendDurationMSec = Math.Max(m_stats.MaxSuspendDurationMSec, _gc.SuspendDurationMSec); + + foreach (var oom in _gc.OOMDetails) + { + m_stats.OOMDetails.Add(oom); + } } m_prvcount = m_gcs.Count; @@ -4690,6 +4695,10 @@ public double GetGCPauseTimePercentage() /// Indicator if PerHeapHistories is present /// public bool HasDetailedGCInfo; + /// + /// OOMs detected in the trace. + /// + public List OOMDetails = new List(); #region private @@ -5004,12 +5013,20 @@ internal static void ProcessCommittedUsage(TraceLoadedDotNetRuntime proc, Commit internal static void ProcessOOMEvent(TraceLoadedDotNetRuntime proc, OOMDetailsTraceEvent oomDetailsTrace) { TraceGC _event = GetLastGC(proc); - _event.OOMDetails.Add(new OOMDetails + if (_event != null) { - GCIndex = oomDetailsTrace.GCIndex, - //Allocated = oomDetailsTrace.Allocated, - Size = oomDetailsTrace.Size, - }); + _event.OOMDetails.Add(new OOMDetails + { + GCIndex = oomDetailsTrace.GCIndex, + Reason = (OOMReason)oomDetailsTrace.Reason, + AllocSize = oomDetailsTrace.AllocSize, + FailureToGetMemory = (FailureGetMemory)oomDetailsTrace.FailureGetMemory, + Size = oomDetailsTrace.Size, + IsLOH = oomDetailsTrace.IsLOH, + MemoryLoad = oomDetailsTrace.MemoryLoad, + AvailablePageFileMB = oomDetailsTrace.AvailablePageFileMB, + }); + } } internal static void ProcessGCDynamicEvent(TraceLoadedDotNetRuntime proc, GCDynamicTraceEvent gcDynamic) diff --git a/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs b/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs index 70cdc87e4..58fe8d99a 100644 --- a/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs +++ b/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs @@ -368,11 +368,11 @@ public sealed class OOMDetailsTraceEvent : GCDynamicEventBase public long GCIndex { get { return BitConverter.ToInt64(DataField, 2); } } public long AllocSize { get { return BitConverter.ToInt64(DataField, 10); } } public short Reason { get { return BitConverter.ToInt16(DataField, 18); } } - public short FailToGetMemory { get { return BitConverter.ToInt16(DataField, 20); } } + public short FailureGetMemory { get { return BitConverter.ToInt16(DataField, 20); } } public long Size { get { return BitConverter.ToInt64(DataField, 22); } } - public short IsLOH { get { return BitConverter.ToInt16(DataField, 30); } } - public int MemoryLoad { get { return BitConverter.ToInt32(DataField, 32); } } - public long AvailablePageFileMB { get { return BitConverter.ToInt64(DataField, 36); } } + public bool IsLOH { get { return BitConverter.ToBoolean(DataField, 30); } } + public int MemoryLoad { get { return BitConverter.ToInt32(DataField, 31); } } + public long AvailablePageFileMB { get { return BitConverter.ToInt64(DataField, 35); } } internal override TraceEventID ID => TraceEventID.Illegal - 12; @@ -389,7 +389,7 @@ internal override string[] PayloadNames { if (_payloadNames == null) { - _payloadNames = new string[] { "Version", "GCIndex", "AllocSize", "Reason", "FailToGetMemory", "Size", "IsLOH", "MemoryLoad", "AvailablePageFileMB" }; + _payloadNames = new string[] { "Version", "GCIndex", "AllocSize", "Reason", "FailureGetMemory", "Size", "IsLOH", "MemoryLoad", "AvailablePageFileMB" }; } return _payloadNames; @@ -404,7 +404,7 @@ internal override IEnumerable> PayloadValues yield return new KeyValuePair("GCIndex", GCIndex); yield return new KeyValuePair("AllocSize", AllocSize); yield return new KeyValuePair("Reason", Reason); - yield return new KeyValuePair("FailToGetMemory", FailToGetMemory); + yield return new KeyValuePair("FailureGetMemory", FailureGetMemory); yield return new KeyValuePair("Size", Size); yield return new KeyValuePair("IsLOH", IsLOH); yield return new KeyValuePair("MemoryLoad", MemoryLoad); @@ -425,7 +425,7 @@ internal override object PayloadValue(int index) case 3: return Reason; case 4: - return FailToGetMemory; + return FailureGetMemory; case 5: return Size; case 6: @@ -441,14 +441,36 @@ internal override object PayloadValue(int index) } } + public enum FailureGetMemory + { + NoFailure = 0, + ReserveSegment = 1, + CommitSegmentBeg = 2, + CommitEphSegment = 3, + GrowTable = 4, + CommitTable = 5, + CommitHeap = 6 + }; + + public enum OOMReason + { + NoFailure = 0, + Budget = 1, + CantCommit = 2, + CantReserve = 3, + LOH = 4, + LowMemory = 5, + UnproductiveFullGC = 6 + } + public sealed class OOMDetails { public long GCIndex { get; internal set; } public long AllocSize { get; internal set; } - public short Reason { get; internal set; } - public short FailToGetMemory { get; internal set; } + public OOMReason Reason { get; internal set; } + public FailureGetMemory FailureToGetMemory { get; internal set; } public long Size { get; internal set; } - public short IsLOH { get; internal set; } + public bool IsLOH { get; internal set; } public int MemoryLoad { get; internal set; } public long AvailablePageFileMB { get; internal set; } } From 7aac9207725aa82e4a4369dd44f00d1c26ff58a5 Mon Sep 17 00:00:00 2001 From: Mukund Raghav Sharma Date: Thu, 30 Jan 2025 12:18:31 -0800 Subject: [PATCH 4/6] Worked on adding the OOM reason to the GCStats page --- src/PerfView/GcStats.cs | 83 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/src/PerfView/GcStats.cs b/src/PerfView/GcStats.cs index 91640b17c..73ab506be 100644 --- a/src/PerfView/GcStats.cs +++ b/src/PerfView/GcStats.cs @@ -2,14 +2,18 @@ using Microsoft.Diagnostics.Tracing.Analysis; using Microsoft.Diagnostics.Tracing.Analysis.GC; using Microsoft.Diagnostics.Tracing.Parsers.Clr; +using Microsoft.Diagnostics.Tracing.Parsers.GCDynamic; using Microsoft.Diagnostics.Utilities; using System; using System.Collections.Generic; +using System.Drawing; using System.IO; using System.Linq; +using System.Net.Http.Headers; using System.Security; using System.Text; using System.Threading; +using System.Windows.Forms; using Utilities; namespace Stats @@ -221,6 +225,85 @@ public static void ToHtml(TextWriter writer, TraceProcess stats, TraceLoadedDotN writer.WriteLine("

View the full list in Excel.

", stats.ProcessID); } + + if (runtime.GC.Stats().OOMDetails.Count > 0) + { + string InterpretOOMReason(OOMReason reason) + { + switch (reason) + { + case OOMReason.Budget: + case OOMReason.CantReserve: + return "OOM was due to an internal .Net error, likely a bug in the GC"; + case OOMReason.CantCommit: + return "Didn't have enough memory to commit"; + case OOMReason.LOH: + return "Didn't have enough memory to allocate an LOH segment"; + case OOMReason.LowMemory: + return "Low on memory during GC"; + case OOMReason.UnproductiveFullGC: + return "Could not do a full GC"; + default: + return reason.ToString(); // shouldn't happen, we handle all cases above + } + } + + string InterpretFailureGetMemory(FailureGetMemory fgm) + { + switch (fgm) + { + case FailureGetMemory.ReserveSegment: + return "Failed to reserve memory"; + case FailureGetMemory.CommitSegmentBeg: + return "Didn't have enough memory to commit beginning of the segment"; + case FailureGetMemory.CommitEphSegment: + return "Didn't have enough memory to commit the new ephemeral segment"; + case FailureGetMemory.GrowTable: + return "Didn't have enough memory to grow the internal GC data structures"; + case FailureGetMemory.CommitTable: + return "Didn't have enough memory to commit the internal GC data structures"; + case FailureGetMemory.CommitHeap: + return "Didn't have enough memory to commit the heap."; + default: + return fgm.ToString(); + } + } + + void WriteOOMHeader() + { + writer.WriteLine(@" + GC Index + Reason + Failure Get Memory Reason + Is LOH + Available Page Memory (MB) + Memory Load (%) + "); + } + + void WriteOOMRow(OOMDetails oomDetails) + { + writer.WriteLine("" + + $"{oomDetails.GCIndex}" + + $"{InterpretOOMReason(oomDetails.Reason)}" + + $"{InterpretFailureGetMemory(oomDetails.FailureToGetMemory)}" + + $"{oomDetails.IsLOH}" + + $"{oomDetails.AvailablePageFileMB}" + + $"{oomDetails.MemoryLoad}" + + ""); + } + + writer.WriteLine("

"); + writer.WriteLine($"

OOM Details for {stats.ProcessID,5}: {stats.Name}

"); + writer.WriteLine("
"); + WriteOOMHeader(); + foreach (var oomDetails in runtime.GC.Stats().OOMDetails) + { + WriteOOMRow(oomDetails); + } + writer.WriteLine("
"); + } + writer.WriteLine("


"); } From 0c5e929b1fee638633d26dcb689b5176918c0e8e Mon Sep 17 00:00:00 2001 From: Mukund Raghav Sharma Date: Thu, 30 Jan 2025 12:26:32 -0800 Subject: [PATCH 5/6] More cosmetic changes --- src/PerfView/GcStats.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/PerfView/GcStats.cs b/src/PerfView/GcStats.cs index 73ab506be..5b356dcf9 100644 --- a/src/PerfView/GcStats.cs +++ b/src/PerfView/GcStats.cs @@ -6,14 +6,11 @@ using Microsoft.Diagnostics.Utilities; using System; using System.Collections.Generic; -using System.Drawing; using System.IO; using System.Linq; -using System.Net.Http.Headers; using System.Security; using System.Text; using System.Threading; -using System.Windows.Forms; using Utilities; namespace Stats @@ -288,7 +285,7 @@ void WriteOOMRow(OOMDetails oomDetails) $"{InterpretOOMReason(oomDetails.Reason)}" + $"{InterpretFailureGetMemory(oomDetails.FailureToGetMemory)}" + $"{oomDetails.IsLOH}" + - $"{oomDetails.AvailablePageFileMB}" + + $"{oomDetails.AvailablePageFileMB.ToString("N0")}" + $"{oomDetails.MemoryLoad}" + ""); } From 6fd20b9c32964e476a9043e48dfed087f8171bfa Mon Sep 17 00:00:00 2001 From: Mukund Raghav Sharma Date: Thu, 30 Jan 2025 12:29:03 -0800 Subject: [PATCH 6/6] Removed redundant change --- src/PerfView/GcStats.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/PerfView/GcStats.cs b/src/PerfView/GcStats.cs index 5b356dcf9..46780f644 100644 --- a/src/PerfView/GcStats.cs +++ b/src/PerfView/GcStats.cs @@ -597,7 +597,6 @@ public static void ToXmlAttribs(TextWriter writer, TraceProcess stats, TraceLoad if (gc.TimingInfo[(int)TraceGC.TimingType.MarkScanFinalization].HasValue) { writer.Write(" MarkScanFinalization=\"{0}\"", gc.TimingInfo[(int)TraceGC.TimingType.MarkScanFinalization].Value); } if (gc.TimingInfo[(int)TraceGC.TimingType.MarkShortWeak].HasValue) { writer.Write(" MarkShortWeak=\"{0}\"", gc.TimingInfo[(int)TraceGC.TimingType.MarkShortWeak].Value); } if (gc.TimingInfo[(int)TraceGC.TimingType.MarkLongWeak].HasValue) { writer.Write(" MarkLongWeak=\"{0}\"", gc.TimingInfo[(int)TraceGC.TimingType.MarkLongWeak].Value); } - if (gc.TimingInfo[(int)TraceGC.TimingType.MarkShortWeak].HasValue) { writer.Write(" MarkShortWeak=\"{0}\"", gc.TimingInfo[(int)TraceGC.TimingType.MarkShortWeak].Value); } if (gc.TimingInfo[(int)TraceGC.TimingType.Plan].HasValue) { writer.Write(" Plan=\"{0}\"", gc.TimingInfo[(int)TraceGC.TimingType.Plan].Value); } if (gc.TimingInfo[(int)TraceGC.TimingType.Relocate].HasValue) { writer.Write(" Relocate=\"{0}\"", gc.TimingInfo[(int)TraceGC.TimingType.Relocate].Value); } if (gc.TimingInfo[(int)TraceGC.TimingType.Compact].HasValue) { writer.Write(" Compact=\"{0}\"", gc.TimingInfo[(int)TraceGC.TimingType.Compact].Value); }