diff --git a/src/PerfView/GcStats.cs b/src/PerfView/GcStats.cs index e0e8b099c..46780f644 100644 --- a/src/PerfView/GcStats.cs +++ b/src/PerfView/GcStats.cs @@ -2,6 +2,7 @@ using Microsoft.Diagnostics.Tracing.Analysis; using Microsoft.Diagnostics.Tracing.Analysis.GC; using Microsoft.Diagnostics.Tracing.Parsers.Clr; +using Microsoft.Diagnostics.Tracing.Parsers.GCDynamic; using Microsoft.Diagnostics.Utilities; using System; using System.Collections.Generic; @@ -221,6 +222,85 @@ public static void ToHtml(TextWriter writer, TraceProcess stats, TraceLoadedDotN writer.WriteLine("

View the full list in Excel.

", stats.ProcessID); } + + if (runtime.GC.Stats().OOMDetails.Count > 0) + { + string InterpretOOMReason(OOMReason reason) + { + switch (reason) + { + case OOMReason.Budget: + case OOMReason.CantReserve: + return "OOM was due to an internal .Net error, likely a bug in the GC"; + case OOMReason.CantCommit: + return "Didn't have enough memory to commit"; + case OOMReason.LOH: + return "Didn't have enough memory to allocate an LOH segment"; + case OOMReason.LowMemory: + return "Low on memory during GC"; + case OOMReason.UnproductiveFullGC: + return "Could not do a full GC"; + default: + return reason.ToString(); // shouldn't happen, we handle all cases above + } + } + + string InterpretFailureGetMemory(FailureGetMemory fgm) + { + switch (fgm) + { + case FailureGetMemory.ReserveSegment: + return "Failed to reserve memory"; + case FailureGetMemory.CommitSegmentBeg: + return "Didn't have enough memory to commit beginning of the segment"; + case FailureGetMemory.CommitEphSegment: + return "Didn't have enough memory to commit the new ephemeral segment"; + case FailureGetMemory.GrowTable: + return "Didn't have enough memory to grow the internal GC data structures"; + case FailureGetMemory.CommitTable: + return "Didn't have enough memory to commit the internal GC data structures"; + case FailureGetMemory.CommitHeap: + return "Didn't have enough memory to commit the heap."; + default: + return fgm.ToString(); + } + } + + void WriteOOMHeader() + { + writer.WriteLine(@" + GC Index + Reason + Failure Get Memory Reason + Is LOH + Available Page Memory (MB) + Memory Load (%) + "); + } + + void WriteOOMRow(OOMDetails oomDetails) + { + writer.WriteLine("" + + $"{oomDetails.GCIndex}" + + $"{InterpretOOMReason(oomDetails.Reason)}" + + $"{InterpretFailureGetMemory(oomDetails.FailureToGetMemory)}" + + $"{oomDetails.IsLOH}" + + $"{oomDetails.AvailablePageFileMB.ToString("N0")}" + + $"{oomDetails.MemoryLoad}" + + ""); + } + + writer.WriteLine("


"); + writer.WriteLine($"

OOM Details for {stats.ProcessID,5}: {stats.Name}

"); + writer.WriteLine("
"); + WriteOOMHeader(); + foreach (var oomDetails in runtime.GC.Stats().OOMDetails) + { + WriteOOMRow(oomDetails); + } + writer.WriteLine("
"); + } + writer.WriteLine("



"); } diff --git a/src/TraceEvent/Computers/TraceManagedProcess.cs b/src/TraceEvent/Computers/TraceManagedProcess.cs index 361c27a02..d21f67f84 100644 --- a/src/TraceEvent/Computers/TraceManagedProcess.cs +++ b/src/TraceEvent/Computers/TraceManagedProcess.cs @@ -879,6 +879,12 @@ internal static void SetupCallbacks(TraceEventDispatcher source) GCStats.ProcessCommittedUsage(stats, committedUsage); }; + source.Clr.GCDynamicEvent.GCOOMDetails += delegate (OOMDetailsTraceEvent oomDetails) + { + var stats = currentManagedProcess(oomDetails.UnderlyingEvent); + GCStats.ProcessOOMEvent(stats, oomDetails); + }; + source.Clr.GCDynamicEvent.GCDynamicTraceEvent += delegate (GCDynamicTraceEvent gcDynamic) { var stats = currentManagedProcess(gcDynamic.UnderlyingEvent); @@ -1738,6 +1744,11 @@ private void Calculate() m_stats.MaxSizePeakMB = Math.Max(m_stats.MaxSizePeakMB, _gc.HeapSizePeakMB); m_stats.MaxAllocRateMBSec = Math.Max(m_stats.MaxAllocRateMBSec, _gc.AllocRateMBSec); m_stats.MaxSuspendDurationMSec = Math.Max(m_stats.MaxSuspendDurationMSec, _gc.SuspendDurationMSec); + + foreach (var oom in _gc.OOMDetails) + { + m_stats.OOMDetails.Add(oom); + } } m_prvcount = m_gcs.Count; @@ -2146,6 +2157,7 @@ public double PromotedMB public CommittedUsage CommittedUsageBefore { get; internal set; } public CommittedUsage CommittedUsageAfter { get; internal set; } + public List OOMDetails { get; internal set; } = new List(); /// /// Memory survival percentage by generation @@ -4683,6 +4695,10 @@ public double GetGCPauseTimePercentage() /// Indicator if PerHeapHistories is present /// public bool HasDetailedGCInfo; + /// + /// OOMs detected in the trace. + /// + public List OOMDetails = new List(); #region private @@ -4994,6 +5010,25 @@ internal static void ProcessCommittedUsage(TraceLoadedDotNetRuntime proc, Commit } } + internal static void ProcessOOMEvent(TraceLoadedDotNetRuntime proc, OOMDetailsTraceEvent oomDetailsTrace) + { + TraceGC _event = GetLastGC(proc); + if (_event != null) + { + _event.OOMDetails.Add(new OOMDetails + { + GCIndex = oomDetailsTrace.GCIndex, + Reason = (OOMReason)oomDetailsTrace.Reason, + AllocSize = oomDetailsTrace.AllocSize, + FailureToGetMemory = (FailureGetMemory)oomDetailsTrace.FailureGetMemory, + Size = oomDetailsTrace.Size, + IsLOH = oomDetailsTrace.IsLOH, + MemoryLoad = oomDetailsTrace.MemoryLoad, + AvailablePageFileMB = oomDetailsTrace.AvailablePageFileMB, + }); + } + } + internal static void ProcessGCDynamicEvent(TraceLoadedDotNetRuntime proc, GCDynamicTraceEvent gcDynamic) { TraceGC _event = GetLastGC(proc); diff --git a/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs b/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs index 43a267cc8..58fe8d99a 100644 --- a/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs +++ b/src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs @@ -32,6 +32,7 @@ public GCDynamicTraceEventParser(TraceEventSource source) : base(source) // They ensure that Dispatch is called so that the specific event handlers are called for each event. ((ITraceParserServices)source).RegisterEventTemplate(GCDynamicTemplate(Dispatch, GCDynamicEventBase.GCDynamicTemplate)); ((ITraceParserServices)source).RegisterEventTemplate(GCDynamicTemplate(Dispatch, GCDynamicEventBase.CommittedUsageTemplate)); + ((ITraceParserServices)source).RegisterEventTemplate(GCDynamicTemplate(Dispatch, GCDynamicEventBase.OOMDetailsTemplate)); } protected override string GetProviderName() @@ -43,7 +44,7 @@ protected internal override void EnumerateTemplates(Func GCCommittedUsage } } + private event Action _OOMDetails; + public event Action GCOOMDetails + { + add + { + _OOMDetails += value; + } + remove + { + _OOMDetails -= value; + } + } + /// /// Responsible for dispatching the event after we determine its type /// and parse it. @@ -98,6 +113,12 @@ private void Dispatch(GCDynamicTraceEventImpl data) _gcCommittedUsage(data.EventPayload as CommittedUsageTraceEvent); } + else if (_OOMDetails != null && + data.eventID == GCDynamicEventBase.OOMDetailsTemplate.ID) + { + _OOMDetails(data.EventPayload as OOMDetailsTraceEvent); + } + else if (_gcDynamicTraceEvent != null && data.EventPayload is GCDynamicTraceEvent) { @@ -160,6 +181,7 @@ protected internal override Delegate Target private readonly CommittedUsageTraceEvent _committedUsageTemplate = new CommittedUsageTraceEvent(); private readonly GCDynamicTraceEvent _gcDynamicTemplate = new GCDynamicTraceEvent(); + private readonly OOMDetailsTraceEvent _oomDetailsTemplate = new OOMDetailsTraceEvent(); /// /// Contains the fully parsed payload of the dynamic event. @@ -173,6 +195,11 @@ public GCDynamicEventBase EventPayload return _committedUsageTemplate.Bind(this); } + else if (eventID == GCDynamicEventBase.OOMDetailsTemplate.ID) + { + return _oomDetailsTemplate.Bind(this); + } + return _gcDynamicTemplate.Bind(this); } } @@ -214,6 +241,11 @@ private void SelectEventMetadata() eventTemplate = GCDynamicEventBase.CommittedUsageTemplate; } + else if (Name.Equals(GCDynamicEventBase.OOMDetailsTemplate.OpcodeName, StringComparison.InvariantCultureIgnoreCase)) + { + eventTemplate = GCDynamicEventBase.OOMDetailsTemplate; + } + SetMetadataFromTemplate(eventTemplate); } @@ -237,6 +269,7 @@ public abstract class GCDynamicEventBase /// internal static readonly GCDynamicTraceEvent GCDynamicTemplate = new GCDynamicTraceEvent(); internal static readonly CommittedUsageTraceEvent CommittedUsageTemplate = new CommittedUsageTraceEvent(); + internal static readonly OOMDetailsTraceEvent OOMDetailsTemplate = new OOMDetailsTraceEvent(); /// /// Metadata that must be specified for each specific type of dynamic event. @@ -329,6 +362,119 @@ internal override IEnumerable> PayloadValues } } + public sealed class OOMDetailsTraceEvent : GCDynamicEventBase + { + public short Version { get { return BitConverter.ToInt16(DataField, 0); } } + public long GCIndex { get { return BitConverter.ToInt64(DataField, 2); } } + public long AllocSize { get { return BitConverter.ToInt64(DataField, 10); } } + public short Reason { get { return BitConverter.ToInt16(DataField, 18); } } + public short FailureGetMemory { get { return BitConverter.ToInt16(DataField, 20); } } + public long Size { get { return BitConverter.ToInt64(DataField, 22); } } + public bool IsLOH { get { return BitConverter.ToBoolean(DataField, 30); } } + public int MemoryLoad { get { return BitConverter.ToInt32(DataField, 31); } } + public long AvailablePageFileMB { get { return BitConverter.ToInt64(DataField, 35); } } + + internal override TraceEventID ID => TraceEventID.Illegal - 12; + + internal override string TaskName => "GC"; + + internal override string OpcodeName => "OOMDetails"; + + internal override string EventName => "GC/OOMDetails"; + + private string[] _payloadNames; + internal override string[] PayloadNames + { + get + { + if (_payloadNames == null) + { + _payloadNames = new string[] { "Version", "GCIndex", "AllocSize", "Reason", "FailureGetMemory", "Size", "IsLOH", "MemoryLoad", "AvailablePageFileMB" }; + } + + return _payloadNames; + } + } + + internal override IEnumerable> PayloadValues + { + get + { + yield return new KeyValuePair("Version", Version); + yield return new KeyValuePair("GCIndex", GCIndex); + yield return new KeyValuePair("AllocSize", AllocSize); + yield return new KeyValuePair("Reason", Reason); + yield return new KeyValuePair("FailureGetMemory", FailureGetMemory); + yield return new KeyValuePair("Size", Size); + yield return new KeyValuePair("IsLOH", IsLOH); + yield return new KeyValuePair("MemoryLoad", MemoryLoad); + yield return new KeyValuePair("AvailablePageFileMB", AvailablePageFileMB); + } + } + + internal override object PayloadValue(int index) + { + switch (index) + { + case 0: + return Version; + case 1: + return GCIndex; + case 2: + return AllocSize; + case 3: + return Reason; + case 4: + return FailureGetMemory; + case 5: + return Size; + case 6: + return IsLOH; + case 7: + return MemoryLoad; + case 8: + return AvailablePageFileMB; + default: + Debug.Assert(false, "Bad field index"); + return null; + } + } + } + + public enum FailureGetMemory + { + NoFailure = 0, + ReserveSegment = 1, + CommitSegmentBeg = 2, + CommitEphSegment = 3, + GrowTable = 4, + CommitTable = 5, + CommitHeap = 6 + }; + + public enum OOMReason + { + NoFailure = 0, + Budget = 1, + CantCommit = 2, + CantReserve = 3, + LOH = 4, + LowMemory = 5, + UnproductiveFullGC = 6 + } + + public sealed class OOMDetails + { + public long GCIndex { get; internal set; } + public long AllocSize { get; internal set; } + public OOMReason Reason { get; internal set; } + public FailureGetMemory FailureToGetMemory { get; internal set; } + public long Size { get; internal set; } + public bool IsLOH { get; internal set; } + public int MemoryLoad { get; internal set; } + public long AvailablePageFileMB { get; internal set; } + } + public sealed class CommittedUsageTraceEvent : GCDynamicEventBase { public short Version { get { return BitConverter.ToInt16(DataField, 0); } } diff --git a/src/TraceEvent/TraceEvent.cs b/src/TraceEvent/TraceEvent.cs index 24dbbf480..dab6ad214 100644 --- a/src/TraceEvent/TraceEvent.cs +++ b/src/TraceEvent/TraceEvent.cs @@ -2972,6 +2972,7 @@ private void ConfirmAllEventsAreInEnumeration() if (string.Equals(GetType().Name, nameof(GCDynamicTraceEventParser), StringComparison.OrdinalIgnoreCase)) { declaredSet.Remove("CommittedUsage"); + declaredSet.Remove("OOMDetails"); } var enumSet = new SortedDictionary(); @@ -3682,7 +3683,8 @@ internal TraceEvent Lookup(TraceEventNativeMethods.EVENT_RECORD* eventRecord) // Make sure that the assert below doesn't fail by checking if _any_ of the event header ids match. bool gcDynamicTemplateEventHeaderMatch = eventRecord->EventHeader.Id == (ushort)GCDynamicEventBase.GCDynamicTemplate.ID || - eventRecord->EventHeader.Id == (ushort)GCDynamicEventBase.CommittedUsageTemplate.ID; + eventRecord->EventHeader.Id == (ushort)GCDynamicEventBase.CommittedUsageTemplate.ID || + eventRecord->EventHeader.Id == (ushort)GCDynamicEventBase.OOMDetailsTemplate.ID; // Ignore the failure for GC dynamic events because they are all // dispatched through the same template and we vary the event ID.