More complete support for AET files

Trace Wizard/Data/ExecutionCall.cs

@@ -52,6 +52,11 @@ public class ExecutionCall
         public string Nest = "";
         public string Function = "";
 
+        public string AE_Program = "";
+        public string AE_Section = "";
+        public string AE_Step = "";
+        public string AE_Action = "";
+
         protected double _duration;
 
         public List<string> Parameters;
@@ -151,8 +156,21 @@ public enum DiffStatus
         SAME,INSERT,DELETE,MODIFIED
     }
 
+    [Flags]
     public enum ExecutionCallType
     {
-        NORMAL,EXTERNAL,CALL, SQL
+        NORMAL = 1, 
+        EXTERNAL = 2, 
+        CALL = 4, 
+        SQL = 8, 
+        AE = 16, 
+        AE_DOWHEN = 32, 
+        AE_DOWHILE = 64, 
+        AE_DOSELECT = 128, 
+        AE_PPC = 256, 
+        AE_SQL = 512, 
+        AE_CALL = 1024, 
+        AE_LOGMSG = 2048, 
+        AE_DOUNTIL = 4096
     }
 }

Trace Wizard/Data/SQLStatement.cs

@@ -49,6 +49,15 @@ public double Duration
                 return ExecTime + FetchTime;
             }
         }
+
+        public bool IsSelectInit
+        {
+            get
+            {
+                return Statement.StartsWith("%SelectInit");
+            }
+        }
+
         public ExecutionCall ParentCall;
         public int FetchCount;
         public string SQLID;
@@ -65,6 +74,7 @@ public double Duration
 
         public int RCNumber;
         public List<SQLBindValue> BindValues = new List<SQLBindValue>();
+        public List<string> BufferData = null;
         public List<String> Tables = new List<string>();
         public SQLType Type;
 
@@ -157,6 +167,39 @@ private void ParseFromClause()
                 Tables.Add(FromClause);
             }
         }
+        internal Dictionary<string,string> GetBufferItems()
+        {
+            Dictionary<string, string> items = new Dictionary<string, string>();
+            List<string> columns = GetBufferColumns();
+            for(var x = 0; x < columns.Count; x++)
+            {
+                items.Add(columns[x], BufferData[x]);
+            }
+            return items;
+        }
+        internal List<string> GetBufferColumns()
+        {
+            var columns = new List<string>();
+
+            /* is this a %Select type query? */
+            if (Statement.StartsWith("%Select",StringComparison.InvariantCultureIgnoreCase))
+            {
+                if (BufferData != null && BufferData.Count > 0)
+                {
+                    Regex getBufferColumns = new Regex(@"%Select(?:Init)?\((.*?)\)");
+                    var bufferCols = getBufferColumns.Match(Statement).Groups[1].Value;
+
+                    Regex colSplit = new Regex(@"([^, ]+)");
+                    foreach (Match match in colSplit.Matches(bufferCols))
+                    {
+                        columns.Add(match.Groups[1].Value);
+                    }
+                }
+            }
+
+
+            return columns;
+        }
 
         private void DetermineType()
         {

Trace Wizard/Processors/AETExecutionPathProcessor.cs

@@ -0,0 +1,65 @@
+using System;
+using System.Linq;
+using System.Text.RegularExpressions;
+using TraceWizard.Data;
+
+namespace TraceWizard.Processors
+{
+    internal class AETExecutionPathProcessor : ITraceProcessor
+    {
+        TraceData _data = null;
+
+        Regex appEngineStart = new Regex(@"-- (\d{2}:\d{2}:\d{2}.\d\d\d) \d{4}-\d{2}-\d{2} Tracing Application Engine program ([^ ]+)");
+        Regex stepMarker = new Regex(@"-- (\d{2}:\d{2}:\d{2}.\d\d\d) (\.+)\(([^.]+)\.([^.]+)\.([^.)]+)\) \((.*?)\)");
+        ExecutionContext ctx = null;
+        int lastIndent = 0;
+
+        public void ProcessLine(string line, long lineNumber)
+        {
+            var match = appEngineStart.Match(line);
+            if (match.Success)
+            {
+                ctx.ContextID = "App Engine " + match.Groups[2].Value;
+                return;
+            }
+
+            match = stepMarker.Match(line);
+            if (match.Success)
+            {
+                ctx.ProcessAECall(match, lineNumber);
+            }
+        }
+
+        public void ProcessorComplete(TraceData data)
+        {
+            _data.ExecutionPath.AddRange(ctx.executionCalls);
+            _data.AllExecutionCalls.AddRange(ctx.allCalls);
+
+            foreach (ExecutionCall call in ctx.allCalls.Where(c => c.Type.HasFlag(ExecutionCallType.SQL)))
+            {
+                var lineNumber = call.StartLine;
+                try
+                {
+                    var sqlStatement = data.SQLStatements.Where(s => s.LineNumber == lineNumber + 1).First();
+
+                    call.SQLStatement = sqlStatement;
+                    sqlStatement.ParentCall = call.Parent;
+                    call.Duration = sqlStatement.Duration;
+                }catch (Exception ex)
+                {
+                    /* this happens durring loops where there was no SQL statement. We need to find a SQL statement prior and clone it and update binds when we capture those */
+                    var dummyStatement = new SQLStatement("SELECT 'X' FROM DUAL");
+                    call.SQLStatement = dummyStatement;
+                    dummyStatement.ParentCall = call.Parent;
+                    call.Duration = dummyStatement.Duration;
+                }
+            }
+        }
+
+        public void ProcessorInit(TraceData data)
+        {
+            ctx = new ExecutionContext("App Engine Trace");
+            _data = data;
+        }
+    }
+}

Trace Wizard/Processors/AETSQLProcessor.cs

@@ -0,0 +1,165 @@
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Globalization;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading;
+using TraceWizard.Data;
+
+namespace TraceWizard.Processors
+{
+    internal class AETSQLProcessor : ITraceProcessor
+    {
+        bool expectingSQL = false;
+        bool expectingBuffers = false;
+        bool expectingBinds = false;
+        StringBuilder currentSQL = new StringBuilder();
+        SQLStatement latestSQLStatement = null;
+        long sqlStartLine = 0;
+        Regex stepHasSQL = new Regex(@"\(Do When\)$|\(Do While\)$|\(Do Select\)$|\(SQL\)$|\(Do Until\)$");
+        Regex bufferBindValue = new Regex(@"--\s+(\d+)\) (.*)");
+        List<SQLStatement> statements = null;
+
+        public void ProcessLine(string line, long lineNumber)
+        {
+            if (stepHasSQL.IsMatch(line))
+            {
+                expectingSQL = true;
+                return;
+            }
+
+            if (line.Equals("-- Buffers:"))
+            {
+                expectingBuffers = true;
+                return;
+            }
+
+            if (line.Equals("-- Bind variables:"))
+            {
+                expectingBinds = true;
+                return;
+            }
+
+            if (expectingBuffers)
+            {
+                var match = bufferBindValue.Match(line);
+                if (match.Success)
+                {
+                    /* add to buffer of previous SQL */
+                    if (latestSQLStatement.BufferData == null)
+                    {
+                        /* init buffer data */
+                        latestSQLStatement.BufferData = new List<string>();
+                    }
+                    latestSQLStatement.BufferData.Add(match.Groups[2].Value);
+                } else
+                {
+                    expectingBuffers = false;
+                    return;
+                }
+            }
+
+            if (expectingBinds)
+            {
+                var match = bufferBindValue.Match(line);
+                if (match.Success)
+                {
+                    /* add to buffer of previous SQL */
+                    if (latestSQLStatement.BindValues == null)
+                    {
+                        /* init buffer data */
+                        latestSQLStatement.BindValues = new List<SQLBindValue>();
+                    }
+                    SQLBindValue bind = new SQLBindValue();
+                    bind.Index = int.Parse(match.Groups[1].Value);
+                    bind.Type = 0;
+                    bind.Length = match.Groups[2].Value.Length;
+                    bind.Value = match.Groups[2].Value;
+
+                    latestSQLStatement.BindValues.Add(bind);
+                }
+                else
+                {
+                    expectingBinds = false;
+                    return;
+                }
+            }
+
+            if (expectingSQL && line.Equals("/") == false)
+            {
+                if (sqlStartLine == 0)
+                {
+                    sqlStartLine = lineNumber;
+                }
+
+                /* dirty check for if we might be in a loop or something and we dont have a SQL where we expect it */
+                if (line.StartsWith("--"))
+                {
+                    /* this isn't where we want to be... false positive for SQL statement */
+                    sqlStartLine = 0;
+                    currentSQL.Clear();
+                    expectingSQL = false;
+                    return;
+                }
+
+                currentSQL.Append(line);
+                return;
+            }
+
+            if (expectingSQL && line.Equals("/"))
+            {
+
+                latestSQLStatement = new SQLStatement(currentSQL.ToString());
+                latestSQLStatement.LineNumber = sqlStartLine;
+
+                statements.Add(latestSQLStatement);
+
+                /* reset state */
+                sqlStartLine = 0;
+                currentSQL.Clear();
+                expectingSQL = false;
+
+            }
+
+            //throw new System.NotImplementedException();
+        }
+
+        public void ProcessorComplete(TraceData data)
+        {
+            //throw new System.NotImplementedException();
+            if (statements.Count == 0)
+            {
+                return;
+            }
+
+            /* Group them all by Where */
+            var sqlByWhereList = data.SQLByWhere;
+
+            var byWheres = statements.Where(p => p.Type != SQLType.INSERT).GroupBy(p => p.WhereClause).Select(g => new SQLByWhere { NumberOfCalls = g.Count(), TotalTime = g.Sum(i => i.Duration), WhereClause = g.Key, HasError = g.Count(p => p.IsError) > 0 ? true : false });
+            foreach (var byW in byWheres)
+            {
+                sqlByWhereList.Add(byW);
+            }
+
+            var sqlByFromList = data.SQLByFrom;
+            var byFroms = statements.Where(p => p.Type == SQLType.SELECT || p.Type == SQLType.DELETE).GroupBy(p => p.FromClause).Select(g => new SQLByFrom { NumberOfCalls = g.Count(), TotalTime = g.Sum(i => i.Duration), FromClause = g.Key, HasError = g.Count(p => p.IsError) > 0 ? true : false });
+            foreach (var byF in byFroms)
+            {
+                sqlByFromList.Add(byF);
+            }
+
+            data.Statistics.Add(new StatisticItem() { Category = "SQL Statements", Label = "Total Count", Value = statements.Count.ToString() });
+            SQLStatement longest = statements.OrderBy(s => s.Duration).Reverse().First();
+            data.Statistics.Add(new StatisticItem() { Category = "SQL Statements", Label = "Longest Execution", Value = longest.Duration.ToString(), Tag = longest });
+            SQLStatement mostFetches = statements.OrderBy(s => s.FetchCount).Reverse().First();
+            data.Statistics.Add(new StatisticItem() { Category = "SQL Statements", Label = "Most Fetches", Value = mostFetches.FetchCount.ToString(), Tag = mostFetches });
+        }
+
+        public void ProcessorInit(TraceData data)
+        {
+            statements = data.SQLStatements;
+            Thread.CurrentThread.CurrentCulture = CultureInfo.CreateSpecificCulture("en-US");
+        }
+    }
+}