diff --git a/src/PerfView/GcStats.cs b/src/PerfView/GcStats.cs
index e0e8b099c..46780f644 100644
--- a/src/PerfView/GcStats.cs
+++ b/src/PerfView/GcStats.cs
@@ -2,6 +2,7 @@
using Microsoft.Diagnostics.Tracing.Analysis;
using Microsoft.Diagnostics.Tracing.Analysis.GC;
using Microsoft.Diagnostics.Tracing.Parsers.Clr;
+using Microsoft.Diagnostics.Tracing.Parsers.GCDynamic;
using Microsoft.Diagnostics.Utilities;
using System;
using System.Collections.Generic;
@@ -221,6 +222,85 @@ public static void ToHtml(TextWriter writer, TraceProcess stats, TraceLoadedDotN
writer.WriteLine("
View the full list in Excel.
", stats.ProcessID);
}
+
+ if (runtime.GC.Stats().OOMDetails.Count > 0)
+ {
+ string InterpretOOMReason(OOMReason reason)
+ {
+ switch (reason)
+ {
+ case OOMReason.Budget:
+ case OOMReason.CantReserve:
+ return "OOM was due to an internal .Net error, likely a bug in the GC";
+ case OOMReason.CantCommit:
+ return "Didn't have enough memory to commit";
+ case OOMReason.LOH:
+ return "Didn't have enough memory to allocate an LOH segment";
+ case OOMReason.LowMemory:
+ return "Low on memory during GC";
+ case OOMReason.UnproductiveFullGC:
+ return "Could not do a full GC";
+ default:
+ return reason.ToString(); // shouldn't happen, we handle all cases above
+ }
+ }
+
+ string InterpretFailureGetMemory(FailureGetMemory fgm)
+ {
+ switch (fgm)
+ {
+ case FailureGetMemory.ReserveSegment:
+ return "Failed to reserve memory";
+ case FailureGetMemory.CommitSegmentBeg:
+ return "Didn't have enough memory to commit beginning of the segment";
+ case FailureGetMemory.CommitEphSegment:
+ return "Didn't have enough memory to commit the new ephemeral segment";
+ case FailureGetMemory.GrowTable:
+ return "Didn't have enough memory to grow the internal GC data structures";
+ case FailureGetMemory.CommitTable:
+ return "Didn't have enough memory to commit the internal GC data structures";
+ case FailureGetMemory.CommitHeap:
+ return "Didn't have enough memory to commit the heap.";
+ default:
+ return fgm.ToString();
+ }
+ }
+
+ void WriteOOMHeader()
+ {
+ writer.WriteLine(@"
+ | GC Index |
+ Reason |
+ Failure Get Memory Reason |
+ Is LOH |
+ Available Page Memory (MB) |
+ Memory Load (%) |
+
");
+ }
+
+ void WriteOOMRow(OOMDetails oomDetails)
+ {
+ writer.WriteLine("" +
+ $"| {oomDetails.GCIndex} | " +
+ $"{InterpretOOMReason(oomDetails.Reason)} | " +
+ $"{InterpretFailureGetMemory(oomDetails.FailureToGetMemory)} | " +
+ $"{oomDetails.IsLOH} | " +
+ $"{oomDetails.AvailablePageFileMB.ToString("N0")} | " +
+ $"{oomDetails.MemoryLoad} | " +
+ "
");
+ }
+
+ writer.WriteLine("
");
+ writer.WriteLine($"");
+ writer.WriteLine("");
+ WriteOOMHeader();
+ foreach (var oomDetails in runtime.GC.Stats().OOMDetails)
+ {
+ WriteOOMRow(oomDetails);
+ }
+ writer.WriteLine("
");
+ }
+
writer.WriteLine("
");
}
diff --git a/src/TraceEvent/Computers/TraceManagedProcess.cs b/src/TraceEvent/Computers/TraceManagedProcess.cs
index 361c27a02..d21f67f84 100644
--- a/src/TraceEvent/Computers/TraceManagedProcess.cs
+++ b/src/TraceEvent/Computers/TraceManagedProcess.cs
@@ -879,6 +879,12 @@ internal static void SetupCallbacks(TraceEventDispatcher source)
GCStats.ProcessCommittedUsage(stats, committedUsage);
};
+ source.Clr.GCDynamicEvent.GCOOMDetails += delegate (OOMDetailsTraceEvent oomDetails)
+ {
+ var stats = currentManagedProcess(oomDetails.UnderlyingEvent);
+ GCStats.ProcessOOMEvent(stats, oomDetails);
+ };
+
source.Clr.GCDynamicEvent.GCDynamicTraceEvent += delegate (GCDynamicTraceEvent gcDynamic)
{
var stats = currentManagedProcess(gcDynamic.UnderlyingEvent);
@@ -1738,6 +1744,11 @@ private void Calculate()
m_stats.MaxSizePeakMB = Math.Max(m_stats.MaxSizePeakMB, _gc.HeapSizePeakMB);
m_stats.MaxAllocRateMBSec = Math.Max(m_stats.MaxAllocRateMBSec, _gc.AllocRateMBSec);
m_stats.MaxSuspendDurationMSec = Math.Max(m_stats.MaxSuspendDurationMSec, _gc.SuspendDurationMSec);
+
+ foreach (var oom in _gc.OOMDetails)
+ {
+ m_stats.OOMDetails.Add(oom);
+ }
}
m_prvcount = m_gcs.Count;
@@ -2146,6 +2157,7 @@ public double PromotedMB
public CommittedUsage CommittedUsageBefore { get; internal set; }
public CommittedUsage CommittedUsageAfter { get; internal set; }
+ public List OOMDetails { get; internal set; } = new List();
///
/// Memory survival percentage by generation
@@ -4683,6 +4695,10 @@ public double GetGCPauseTimePercentage()
/// Indicator if PerHeapHistories is present
///
public bool HasDetailedGCInfo;
+ ///
+ /// OOMs detected in the trace.
+ ///
+ public List OOMDetails = new List();
#region private
@@ -4994,6 +5010,25 @@ internal static void ProcessCommittedUsage(TraceLoadedDotNetRuntime proc, Commit
}
}
+ internal static void ProcessOOMEvent(TraceLoadedDotNetRuntime proc, OOMDetailsTraceEvent oomDetailsTrace)
+ {
+ TraceGC _event = GetLastGC(proc);
+ if (_event != null)
+ {
+ _event.OOMDetails.Add(new OOMDetails
+ {
+ GCIndex = oomDetailsTrace.GCIndex,
+ Reason = (OOMReason)oomDetailsTrace.Reason,
+ AllocSize = oomDetailsTrace.AllocSize,
+ FailureToGetMemory = (FailureGetMemory)oomDetailsTrace.FailureGetMemory,
+ Size = oomDetailsTrace.Size,
+ IsLOH = oomDetailsTrace.IsLOH,
+ MemoryLoad = oomDetailsTrace.MemoryLoad,
+ AvailablePageFileMB = oomDetailsTrace.AvailablePageFileMB,
+ });
+ }
+ }
+
internal static void ProcessGCDynamicEvent(TraceLoadedDotNetRuntime proc, GCDynamicTraceEvent gcDynamic)
{
TraceGC _event = GetLastGC(proc);
diff --git a/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs b/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs
index 43a267cc8..58fe8d99a 100644
--- a/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs
+++ b/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs
@@ -32,6 +32,7 @@ public GCDynamicTraceEventParser(TraceEventSource source) : base(source)
// They ensure that Dispatch is called so that the specific event handlers are called for each event.
((ITraceParserServices)source).RegisterEventTemplate(GCDynamicTemplate(Dispatch, GCDynamicEventBase.GCDynamicTemplate));
((ITraceParserServices)source).RegisterEventTemplate(GCDynamicTemplate(Dispatch, GCDynamicEventBase.CommittedUsageTemplate));
+ ((ITraceParserServices)source).RegisterEventTemplate(GCDynamicTemplate(Dispatch, GCDynamicEventBase.OOMDetailsTemplate));
}
protected override string GetProviderName()
@@ -43,7 +44,7 @@ protected internal override void EnumerateTemplates(Func GCCommittedUsage
}
}
+ private event Action _OOMDetails;
+ public event Action GCOOMDetails
+ {
+ add
+ {
+ _OOMDetails += value;
+ }
+ remove
+ {
+ _OOMDetails -= value;
+ }
+ }
+
///
/// Responsible for dispatching the event after we determine its type
/// and parse it.
@@ -98,6 +113,12 @@ private void Dispatch(GCDynamicTraceEventImpl data)
_gcCommittedUsage(data.EventPayload as CommittedUsageTraceEvent);
}
+ else if (_OOMDetails != null &&
+ data.eventID == GCDynamicEventBase.OOMDetailsTemplate.ID)
+ {
+ _OOMDetails(data.EventPayload as OOMDetailsTraceEvent);
+ }
+
else if (_gcDynamicTraceEvent != null &&
data.EventPayload is GCDynamicTraceEvent)
{
@@ -160,6 +181,7 @@ protected internal override Delegate Target
private readonly CommittedUsageTraceEvent _committedUsageTemplate = new CommittedUsageTraceEvent();
private readonly GCDynamicTraceEvent _gcDynamicTemplate = new GCDynamicTraceEvent();
+ private readonly OOMDetailsTraceEvent _oomDetailsTemplate = new OOMDetailsTraceEvent();
///
/// Contains the fully parsed payload of the dynamic event.
@@ -173,6 +195,11 @@ public GCDynamicEventBase EventPayload
return _committedUsageTemplate.Bind(this);
}
+ else if (eventID == GCDynamicEventBase.OOMDetailsTemplate.ID)
+ {
+ return _oomDetailsTemplate.Bind(this);
+ }
+
return _gcDynamicTemplate.Bind(this);
}
}
@@ -214,6 +241,11 @@ private void SelectEventMetadata()
eventTemplate = GCDynamicEventBase.CommittedUsageTemplate;
}
+ else if (Name.Equals(GCDynamicEventBase.OOMDetailsTemplate.OpcodeName, StringComparison.InvariantCultureIgnoreCase))
+ {
+ eventTemplate = GCDynamicEventBase.OOMDetailsTemplate;
+ }
+
SetMetadataFromTemplate(eventTemplate);
}
@@ -237,6 +269,7 @@ public abstract class GCDynamicEventBase
///
internal static readonly GCDynamicTraceEvent GCDynamicTemplate = new GCDynamicTraceEvent();
internal static readonly CommittedUsageTraceEvent CommittedUsageTemplate = new CommittedUsageTraceEvent();
+ internal static readonly OOMDetailsTraceEvent OOMDetailsTemplate = new OOMDetailsTraceEvent();
///
/// Metadata that must be specified for each specific type of dynamic event.
@@ -329,6 +362,119 @@ internal override IEnumerable> PayloadValues
}
}
+ public sealed class OOMDetailsTraceEvent : GCDynamicEventBase
+ {
+ public short Version { get { return BitConverter.ToInt16(DataField, 0); } }
+ public long GCIndex { get { return BitConverter.ToInt64(DataField, 2); } }
+ public long AllocSize { get { return BitConverter.ToInt64(DataField, 10); } }
+ public short Reason { get { return BitConverter.ToInt16(DataField, 18); } }
+ public short FailureGetMemory { get { return BitConverter.ToInt16(DataField, 20); } }
+ public long Size { get { return BitConverter.ToInt64(DataField, 22); } }
+ public bool IsLOH { get { return BitConverter.ToBoolean(DataField, 30); } }
+ public int MemoryLoad { get { return BitConverter.ToInt32(DataField, 31); } }
+ public long AvailablePageFileMB { get { return BitConverter.ToInt64(DataField, 35); } }
+
+ internal override TraceEventID ID => TraceEventID.Illegal - 12;
+
+ internal override string TaskName => "GC";
+
+ internal override string OpcodeName => "OOMDetails";
+
+ internal override string EventName => "GC/OOMDetails";
+
+ private string[] _payloadNames;
+ internal override string[] PayloadNames
+ {
+ get
+ {
+ if (_payloadNames == null)
+ {
+ _payloadNames = new string[] { "Version", "GCIndex", "AllocSize", "Reason", "FailureGetMemory", "Size", "IsLOH", "MemoryLoad", "AvailablePageFileMB" };
+ }
+
+ return _payloadNames;
+ }
+ }
+
+ internal override IEnumerable> PayloadValues
+ {
+ get
+ {
+ yield return new KeyValuePair("Version", Version);
+ yield return new KeyValuePair("GCIndex", GCIndex);
+ yield return new KeyValuePair("AllocSize", AllocSize);
+ yield return new KeyValuePair("Reason", Reason);
+ yield return new KeyValuePair("FailureGetMemory", FailureGetMemory);
+ yield return new KeyValuePair("Size", Size);
+ yield return new KeyValuePair("IsLOH", IsLOH);
+ yield return new KeyValuePair("MemoryLoad", MemoryLoad);
+ yield return new KeyValuePair("AvailablePageFileMB", AvailablePageFileMB);
+ }
+ }
+
+ internal override object PayloadValue(int index)
+ {
+ switch (index)
+ {
+ case 0:
+ return Version;
+ case 1:
+ return GCIndex;
+ case 2:
+ return AllocSize;
+ case 3:
+ return Reason;
+ case 4:
+ return FailureGetMemory;
+ case 5:
+ return Size;
+ case 6:
+ return IsLOH;
+ case 7:
+ return MemoryLoad;
+ case 8:
+ return AvailablePageFileMB;
+ default:
+ Debug.Assert(false, "Bad field index");
+ return null;
+ }
+ }
+ }
+
+ public enum FailureGetMemory
+ {
+ NoFailure = 0,
+ ReserveSegment = 1,
+ CommitSegmentBeg = 2,
+ CommitEphSegment = 3,
+ GrowTable = 4,
+ CommitTable = 5,
+ CommitHeap = 6
+ };
+
+ public enum OOMReason
+ {
+ NoFailure = 0,
+ Budget = 1,
+ CantCommit = 2,
+ CantReserve = 3,
+ LOH = 4,
+ LowMemory = 5,
+ UnproductiveFullGC = 6
+ }
+
+ public sealed class OOMDetails
+ {
+ public long GCIndex { get; internal set; }
+ public long AllocSize { get; internal set; }
+ public OOMReason Reason { get; internal set; }
+ public FailureGetMemory FailureToGetMemory { get; internal set; }
+ public long Size { get; internal set; }
+ public bool IsLOH { get; internal set; }
+ public int MemoryLoad { get; internal set; }
+ public long AvailablePageFileMB { get; internal set; }
+ }
+
public sealed class CommittedUsageTraceEvent : GCDynamicEventBase
{
public short Version { get { return BitConverter.ToInt16(DataField, 0); } }
diff --git a/src/TraceEvent/TraceEvent.cs b/src/TraceEvent/TraceEvent.cs
index 24dbbf480..dab6ad214 100644
--- a/src/TraceEvent/TraceEvent.cs
+++ b/src/TraceEvent/TraceEvent.cs
@@ -2972,6 +2972,7 @@ private void ConfirmAllEventsAreInEnumeration()
if (string.Equals(GetType().Name, nameof(GCDynamicTraceEventParser), StringComparison.OrdinalIgnoreCase))
{
declaredSet.Remove("CommittedUsage");
+ declaredSet.Remove("OOMDetails");
}
var enumSet = new SortedDictionary();
@@ -3682,7 +3683,8 @@ internal TraceEvent Lookup(TraceEventNativeMethods.EVENT_RECORD* eventRecord)
// Make sure that the assert below doesn't fail by checking if _any_ of the event header ids match.
bool gcDynamicTemplateEventHeaderMatch =
eventRecord->EventHeader.Id == (ushort)GCDynamicEventBase.GCDynamicTemplate.ID ||
- eventRecord->EventHeader.Id == (ushort)GCDynamicEventBase.CommittedUsageTemplate.ID;
+ eventRecord->EventHeader.Id == (ushort)GCDynamicEventBase.CommittedUsageTemplate.ID ||
+ eventRecord->EventHeader.Id == (ushort)GCDynamicEventBase.OOMDetailsTemplate.ID;
// Ignore the failure for GC dynamic events because they are all
// dispatched through the same template and we vary the event ID.