Modules/businessdev.ALbuild.Apps/Resources/CallGraph/CallGraphEngine.cs
|
// Call graph engine for businessdev.ALbuild. Compiled with Add-Type on Windows PowerShell 5.1 (C# 5)
// and PowerShell 7, so it uses no newer language features. using System; using System.Collections; using System.Collections.Generic; using System.Globalization; using System.IO; using System.IO.Compression; using System.Linq; using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; using System.Xml; namespace ALbuild.CallGraph { /// <summary>Minimal JSON reader and writer; the graph files are JSON lines of flat records.</summary> public static class Json { public static object Parse(string text) { int i = 0; object value = ReadValue(text, ref i); return value; } static void SkipWs(string s, ref int i) { while (i < s.Length && char.IsWhiteSpace(s[i])) i++; } static object ReadValue(string s, ref int i) { SkipWs(s, ref i); if (i >= s.Length) throw new FormatException("Unexpected end of JSON."); char c = s[i]; if (c == '{') { var obj = new Dictionary<string, object>(StringComparer.Ordinal); i++; SkipWs(s, ref i); if (s[i] == '}') { i++; return obj; } while (true) { SkipWs(s, ref i); string key = ReadString(s, ref i); SkipWs(s, ref i); if (s[i] != ':') throw new FormatException("Expected ':' at " + i); i++; obj[key] = ReadValue(s, ref i); SkipWs(s, ref i); if (s[i] == ',') { i++; continue; } if (s[i] == '}') { i++; return obj; } throw new FormatException("Expected ',' or '}' at " + i); } } if (c == '[') { var list = new List<object>(); i++; SkipWs(s, ref i); if (s[i] == ']') { i++; return list; } while (true) { list.Add(ReadValue(s, ref i)); SkipWs(s, ref i); if (s[i] == ',') { i++; continue; } if (s[i] == ']') { i++; return list; } throw new FormatException("Expected ',' or ']' at " + i); } } if (c == '"') return ReadString(s, ref i); if (string.CompareOrdinal(s, i, "true", 0, 4) == 0) { i += 4; return true; } if (string.CompareOrdinal(s, i, "false", 0, 5) == 0) { i += 5; return false; } if (string.CompareOrdinal(s, i, "null", 0, 4) == 0) { i += 4; return null; } int start = i; while (i < s.Length && "+-0123456789.eE".IndexOf(s[i]) >= 0) i++; string num = s.Substring(start, i - start); long l; if (long.TryParse(num, NumberStyles.Integer, CultureInfo.InvariantCulture, out l)) return l; return double.Parse(num, CultureInfo.InvariantCulture); } static string ReadString(string s, ref int i) { if (s[i] != '"') throw new FormatException("Expected string at " + i); i++; var sb = new StringBuilder(); while (true) { char c = s[i++]; if (c == '"') return sb.ToString(); if (c != '\\') { sb.Append(c); continue; } char e = s[i++]; switch (e) { case '"': sb.Append('"'); break; case '\\': sb.Append('\\'); break; case '/': sb.Append('/'); break; case 'b': sb.Append('\b'); break; case 'f': sb.Append('\f'); break; case 'n': sb.Append('\n'); break; case 'r': sb.Append('\r'); break; case 't': sb.Append('\t'); break; case 'u': sb.Append((char)int.Parse(s.Substring(i, 4), NumberStyles.HexNumber, CultureInfo.InvariantCulture)); i += 4; break; default: throw new FormatException("Bad escape at " + i); } } } public static string Serialize(object value) { var sb = new StringBuilder(); Write(sb, value); return sb.ToString(); } /// <summary>Indented JSON for files people read and edit (callgraph.architecture.json); no HTML escaping.</summary> public static string Pretty(object value) { var sb = new StringBuilder(); WritePretty(sb, value, 0); return sb.ToString(); } static void WritePretty(StringBuilder sb, object value, int depth) { string pad = new string(' ', (depth + 1) * 2), close = new string(' ', depth * 2); var dict = value as IDictionary; if (dict != null) { if (dict.Count == 0) { sb.Append("{}"); return; } sb.Append("{\n"); bool first = true; foreach (DictionaryEntry kv in dict) { if (!first) sb.Append(",\n"); first = false; sb.Append(pad); PlainString(sb, Convert.ToString(kv.Key, CultureInfo.InvariantCulture)); sb.Append(": "); WritePretty(sb, kv.Value, depth + 1); } sb.Append('\n').Append(close).Append('}'); return; } var list = value as IEnumerable; if (list != null && !(value is string)) { var items = list.Cast<object>().ToList(); if (items.Count == 0) { sb.Append("[]"); return; } // Short lists of strings stay on one line (paths, patterns). if (items.All(i => i is string) && items.Sum(i => ((string)i).Length) < 100) { sb.Append('['); for (int i = 0; i < items.Count; i++) { if (i > 0) sb.Append(", "); PlainString(sb, (string)items[i]); } sb.Append(']'); return; } sb.Append("[\n"); for (int i = 0; i < items.Count; i++) { if (i > 0) sb.Append(",\n"); sb.Append(pad); WritePretty(sb, items[i], depth + 1); } sb.Append('\n').Append(close).Append(']'); return; } var str = value as string; if (str != null) { PlainString(sb, str); return; } Write(sb, value); } static void PlainString(StringBuilder sb, string s) { sb.Append('"'); foreach (char c in s) { switch (c) { case '"': sb.Append("\\\""); break; case '\\': sb.Append("\\\\"); break; case '\n': sb.Append("\\n"); break; case '\r': sb.Append("\\r"); break; case '\t': sb.Append("\\t"); break; default: if (c < 0x20) sb.Append("\\u").Append(((int)c).ToString("x4", CultureInfo.InvariantCulture)); else sb.Append(c); break; } } sb.Append('"'); } public static void Write(StringBuilder sb, object value) { if (value == null) { sb.Append("null"); return; } var str = value as string; if (str != null) { WriteString(sb, str); return; } if (value is bool) { sb.Append((bool)value ? "true" : "false"); return; } if (value is int || value is long) { sb.Append(Convert.ToInt64(value).ToString(CultureInfo.InvariantCulture)); return; } if (value is double || value is float || value is decimal) { sb.Append(Convert.ToDouble(value).ToString("0.###", CultureInfo.InvariantCulture)); return; } var dict = value as IDictionary; if (dict != null) { sb.Append('{'); bool first = true; foreach (DictionaryEntry kv in dict) { if (!first) sb.Append(','); first = false; WriteString(sb, Convert.ToString(kv.Key, CultureInfo.InvariantCulture)); sb.Append(':'); Write(sb, kv.Value); } sb.Append('}'); return; } var list = value as IEnumerable; if (list != null) { sb.Append('['); bool first = true; foreach (var item in list) { if (!first) sb.Append(','); first = false; Write(sb, item); } sb.Append(']'); return; } WriteString(sb, Convert.ToString(value, CultureInfo.InvariantCulture)); } public static void WriteString(StringBuilder sb, string s) { sb.Append('"'); foreach (char c in s) { switch (c) { case '"': sb.Append("\\\""); break; case '\\': sb.Append("\\\\"); break; case '\n': sb.Append("\\n"); break; case '\r': sb.Append("\\r"); break; case '\t': sb.Append("\\t"); break; case '<': sb.Append("\\u003c"); break; // safe to embed in an HTML <script> case '>': sb.Append("\\u003e"); break; case '&': sb.Append("\\u0026"); break; default: if (c < 0x20) sb.Append("\\u").Append(((int)c).ToString("x4", CultureInfo.InvariantCulture)); else sb.Append(c); break; } } sb.Append('"'); } } /// <summary>The manifest of a .app package and whether it carries AL source.</summary> public class AppManifest { public string Id; public string Name; public string Publisher; public string Version; public bool HasSource; public bool IsRuntimePackage; public string ManifestXml; public List<AppReference> Dependencies = new List<AppReference>(); public string Path; } public class AppReference { public string Id; public string Name; public string Publisher; } /// <summary>Reads .app packages, including ReadyToRun packages that wrap the real app as an inner .app.</summary> public static class AppPackage { static ZipArchive OpenZip(byte[] data) { int offset = -1; for (int i = 0; i + 3 < data.Length && i < 4096; i++) { if (data[i] == 0x50 && data[i + 1] == 0x4B && data[i + 2] == 0x03 && data[i + 3] == 0x04) { offset = i; break; } } if (offset < 0) return null; return new ZipArchive(new MemoryStream(data, offset, data.Length - offset, false), ZipArchiveMode.Read); } static byte[] ReadEntry(ZipArchiveEntry entry) { using (var s = entry.Open()) using (var ms = new MemoryStream()) { s.CopyTo(ms); return ms.ToArray(); } } /// <summary>Opens the archive that holds NavxManifest.xml, descending into an inner .app when needed.</summary> static ZipArchive OpenSourceArchive(byte[] data, int depth) { var zip = OpenZip(data); if (zip == null) return null; if (zip.GetEntry("NavxManifest.xml") != null) return zip; if (depth > 2) return null; foreach (var entry in zip.Entries) { if (entry.FullName.IndexOf('/') < 0 && entry.FullName.EndsWith(".app", StringComparison.OrdinalIgnoreCase)) { var inner = OpenSourceArchive(ReadEntry(entry), depth + 1); if (inner != null) return inner; } } return null; } /// <summary>Reads the manifest; null for a package that is not a readable zip (e.g. an encrypted runtime package).</summary> public static AppManifest ReadManifest(string path) { byte[] data = File.ReadAllBytes(path); var zip = OpenSourceArchive(data, 0); if (zip == null) return null; using (zip) { var m = new AppManifest(); m.Path = path; using (var r = new StreamReader(zip.GetEntry("NavxManifest.xml").Open(), Encoding.UTF8)) { m.ManifestXml = r.ReadToEnd(); } var doc = new XmlDocument(); doc.LoadXml(m.ManifestXml); foreach (XmlNode node in doc.DocumentElement.ChildNodes) { if (node.LocalName == "App" && node.Attributes != null) { m.Id = Attr(node, "Id"); m.Name = Attr(node, "Name"); m.Publisher = Attr(node, "Publisher"); m.Version = Attr(node, "Version"); } if (node.LocalName == "Dependencies") { foreach (XmlNode d in node.ChildNodes) { string id = Attr(d, "Id"); if (string.IsNullOrEmpty(id)) continue; var dep = new AppReference(); dep.Id = id.ToLowerInvariant(); dep.Name = Attr(d, "Name"); dep.Publisher = Attr(d, "Publisher"); m.Dependencies.Add(dep); } } } m.HasSource = zip.Entries.Any(e => e.FullName.EndsWith(".al", StringComparison.OrdinalIgnoreCase)); if (m.Id != null) m.Id = m.Id.ToLowerInvariant(); return m; } } static string Attr(XmlNode node, string name) { if (node == null || node.Attributes == null) return null; var a = node.Attributes[name]; return a == null ? null : a.Value; } static string LongPath(string path) { string full = System.IO.Path.GetFullPath(path); if (Environment.OSVersion.Platform != PlatformID.Win32NT || full.StartsWith(@"\\?\", StringComparison.Ordinal)) return full; if (full.StartsWith(@"\\", StringComparison.Ordinal)) return @"\\?\UNC\" + full.Substring(2); return @"\\?\" + full; } /// <summary> /// Writes the package's .al files under destination and synthesizes the app.json the AL compiler needs /// from the manifest. Returns the number of source files written (0 = no source). /// </summary> public static int ExtractSource(string path, string destination) { byte[] data = File.ReadAllBytes(path); var zip = OpenSourceArchive(data, 0); if (zip == null) return 0; int count = 0; using (zip) { string root = LongPath(destination); Directory.CreateDirectory(root); foreach (var entry in zip.Entries) { if (!entry.FullName.EndsWith(".al", StringComparison.OrdinalIgnoreCase)) continue; string relative = entry.FullName.Replace('/', System.IO.Path.DirectorySeparatorChar); if (relative.Contains("..")) continue; string target = System.IO.Path.Combine(root, relative); Directory.CreateDirectory(System.IO.Path.GetDirectoryName(target)); using (var src = entry.Open()) using (var dst = File.Create(target)) { src.CopyTo(dst); } count++; } string manifestXml; using (var r = new StreamReader(zip.GetEntry("NavxManifest.xml").Open(), Encoding.UTF8)) { manifestXml = r.ReadToEnd(); } File.WriteAllText(System.IO.Path.Combine(root, "app.json"), AppJsonFromManifest(manifestXml), new UTF8Encoding(false)); } return count; } /// <summary>An app.json equivalent of a NavxManifest.xml, enough for the compiler to resolve the app.</summary> public static string AppJsonFromManifest(string manifestXml) { var doc = new XmlDocument(); doc.LoadXml(manifestXml); var app = new Dictionary<string, object>(); var deps = new List<object>(); var ranges = new List<object>(); var features = new List<object>(); var symbols = new List<object>(); foreach (XmlNode node in doc.DocumentElement.ChildNodes) { switch (node.LocalName) { case "App": app["id"] = Attr(node, "Id"); app["name"] = Attr(node, "Name"); app["publisher"] = Attr(node, "Publisher"); app["version"] = Attr(node, "Version"); app["platform"] = Attr(node, "Platform") ?? "1.0.0.0"; if (!string.IsNullOrEmpty(Attr(node, "Application"))) app["application"] = Attr(node, "Application"); if (!string.IsNullOrEmpty(Attr(node, "Runtime"))) app["runtime"] = Attr(node, "Runtime"); break; case "IdRanges": foreach (XmlNode r in node.ChildNodes) { int from, to; if (int.TryParse(Attr(r, "MinObjectId"), out from) && int.TryParse(Attr(r, "MaxObjectId"), out to)) { var range = new Dictionary<string, object>(); range["from"] = from; range["to"] = to; ranges.Add(range); } } break; case "Dependencies": foreach (XmlNode d in node.ChildNodes) { if (Attr(d, "Id") == null) continue; var dep = new Dictionary<string, object>(); dep["id"] = Attr(d, "Id"); dep["name"] = Attr(d, "Name"); dep["publisher"] = Attr(d, "Publisher"); dep["version"] = Attr(d, "MinVersion") ?? Attr(d, "Version") ?? "0.0.0.0"; deps.Add(dep); } break; case "Features": foreach (XmlNode f in node.ChildNodes) if (!string.IsNullOrEmpty(f.InnerText)) features.Add(f.InnerText.Trim()); break; case "PreprocessorSymbols": foreach (XmlNode p in node.ChildNodes) { string name = Attr(p, "Name") ?? p.InnerText; if (!string.IsNullOrEmpty(name)) symbols.Add(name.Trim()); } break; } } app["idRanges"] = ranges; app["dependencies"] = deps; app["target"] = "OnPrem"; if (features.Count > 0) app["features"] = features; if (symbols.Count > 0) app["preprocessorSymbols"] = symbols; return Json.Serialize(app); } } public class Node { public string Id; public string App; public string ObjectKey; public string Kind; public string ObjectType; public int ObjectId; public string ObjectName; public string Member; public string SourcePath; public int Line; public string Scope; public string Access; public string EventRole; public bool Obsolete; public bool Test; public string Elevation; // None, InherentPermissions, InherentEntitlements, Both public bool NonDebuggable; } public class Edge { public string Caller; public string Callee; public string Kind; public string Confidence; public string SourcePath; public int Line; } public class Graph { public Dictionary<string, Node> Nodes = new Dictionary<string, Node>(StringComparer.Ordinal); public List<Edge> Edges = new List<Edge>(); public HashSet<string> Subscribers = new HashSet<string>(StringComparer.Ordinal); public HashSet<string> Publishers = new HashSet<string>(StringComparer.Ordinal); public List<string[]> Implementations = new List<string[]>(); public Dictionary<string, string> AppNames = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase); public const string BuiltInApp = "00000000-0000-0000-0000-000000000000"; /// <summary>The app id inside a node id of the form "al . {app} / {type} {id} {name} # {member}".</summary> public static string AppOf(string id) { if (id == null || !id.StartsWith("al . ", StringComparison.Ordinal)) return ""; int slash = id.IndexOf(" / ", 5, StringComparison.Ordinal); return slash < 0 ? "" : id.Substring(5, slash - 5).ToLowerInvariant(); } public static string ObjectKeyOf(string id) { int hash = id.IndexOf(" # ", StringComparison.Ordinal); return hash < 0 ? id : id.Substring(0, hash); } static string Str(Dictionary<string, object> v, string key) { object o; return v.TryGetValue(key, out o) && o != null ? Convert.ToString(o, CultureInfo.InvariantCulture) : null; } static int Int(Dictionary<string, object> v, string key) { object o; return v.TryGetValue(key, out o) && o != null ? Convert.ToInt32(o, CultureInfo.InvariantCulture) : 0; } static bool Bool(Dictionary<string, object> v, string key) { object o; return v.TryGetValue(key, out o) && o is bool && (bool)o; } public static Graph Load(string path) { var g = new Graph(); foreach (string line in File.ReadLines(path)) { if (line.Length == 0) continue; var rec = Json.Parse(line) as Dictionary<string, object>; if (rec == null) continue; string t = Str(rec, "t"); var v = rec.ContainsKey("v") ? rec["v"] as Dictionary<string, object> : null; if (v == null) continue; switch (t) { case "header": string appId = Str(v, "appId"); if (appId != null) g.AppNames[appId.ToLowerInvariant()] = Str(v, "appName"); break; case "node": var n = new Node(); n.Id = Str(v, "id"); n.App = AppOf(n.Id); n.ObjectKey = ObjectKeyOf(n.Id); n.Kind = Str(v, "kind"); n.ObjectId = Int(v, "objectId"); n.ObjectName = Str(v, "objectName"); n.Member = Str(v, "memberName"); n.SourcePath = Str(v, "sourcePath"); n.Line = Int(v, "line"); n.Scope = Str(v, "scope"); n.Access = Str(v, "accessibility"); n.EventRole = Str(v, "eventRole"); n.Obsolete = Bool(v, "isObsolete"); n.Elevation = Str(v, "elevation"); n.NonDebuggable = Bool(v, "isNonDebuggable"); n.ObjectType = ObjectTypeOf(n.ObjectKey); if (n.Id != null) g.Nodes[n.Id] = n; break; case "edge": var e = new Edge(); e.Caller = Str(v, "caller"); e.Callee = Str(v, "callee"); e.Kind = Str(v, "kind"); e.Confidence = Str(v, "confidence"); e.SourcePath = Str(v, "sourcePath"); e.Line = Int(v, "line"); g.Edges.Add(e); break; case "sub": if (Str(v, "subscriber") != null) g.Subscribers.Add(Str(v, "subscriber")); break; case "pub": if (Str(v, "publisher") != null) g.Publishers.Add(Str(v, "publisher")); break; case "impl": g.Implementations.Add(new[] { Str(v, "codeunit"), Str(v, "interfaceName") }); break; } } return g; } /// <summary>"Codeunit" out of "al . {app} / Codeunit 50100 Name".</summary> public static string ObjectTypeOf(string objectKey) { int slash = objectKey.IndexOf(" / ", StringComparison.Ordinal); if (slash < 0) return ""; int space = objectKey.IndexOf(' ', slash + 3); return space < 0 ? objectKey.Substring(slash + 3) : objectKey.Substring(slash + 3, space - slash - 3); } } /// <summary>Statistics of a prune run.</summary> public class PruneResult { public int Nodes; public int Edges; public int SourceNodes; public int SourceEdges; } public static class Pruner { static string FieldValue(string line, string field) { string marker = "\"" + field + "\":\""; int start = line.IndexOf(marker, StringComparison.Ordinal); if (start < 0) return null; start += marker.Length; var sb = new StringBuilder(); for (int i = start; i < line.Length; i++) { char c = line[i]; if (c == '\\' && i + 1 < line.Length) { sb.Append(line[i + 1]); i++; continue; } if (c == '"') break; sb.Append(c); } return sb.ToString(); } static string TypeOf(string line) { if (!line.StartsWith("{\"t\":\"", StringComparison.Ordinal)) return ""; int end = line.IndexOf('"', 6); return end < 0 ? "" : line.Substring(6, end - 6); } /// <summary> /// Keeps the own apps and their boundary: every node of an own app, every edge with an own caller or /// callee, and the foreign nodes (plus their objects) those edges touch. Microsoft's internals - the /// bulk of a whole-corpus graph - are dropped. /// </summary> public static PruneResult Prune(string sourcePath, string targetPath, string[] ownApps) { var own = new HashSet<string>(ownApps.Select(a => a.ToLowerInvariant()), StringComparer.Ordinal); var needed = new HashSet<string>(StringComparer.Ordinal); var headers = new List<string>(); var keep = new List<string>(); var result = new PruneResult(); foreach (string line in File.ReadLines(sourcePath)) { string t = TypeOf(line); switch (t) { case "header": headers.Add(line); break; case "node": result.SourceNodes++; string id = FieldValue(line, "id"); if (id != null && own.Contains(Graph.AppOf(id))) needed.Add(id); break; case "edge": result.SourceEdges++; string caller = FieldValue(line, "caller"); string callee = FieldValue(line, "callee"); if (caller == null || callee == null) break; if (own.Contains(Graph.AppOf(caller)) || own.Contains(Graph.AppOf(callee))) { keep.Add(line); needed.Add(caller); needed.Add(callee); needed.Add(Graph.ObjectKeyOf(caller)); needed.Add(Graph.ObjectKeyOf(callee)); result.Edges++; } break; case "sub": if (own.Contains(Graph.AppOf(FieldValue(line, "subscriber")))) keep.Add(line); break; case "pub": if (own.Contains(Graph.AppOf(FieldValue(line, "publisher")))) keep.Add(line); break; case "impl": if (own.Contains(Graph.AppOf(FieldValue(line, "codeunit")))) keep.Add(line); break; case "icall": if (own.Contains(Graph.AppOf(FieldValue(line, "caller")))) keep.Add(line); break; } } using (var w = new StreamWriter(targetPath, false, new UTF8Encoding(false))) { w.NewLine = "\n"; foreach (var h in headers) w.WriteLine(h); foreach (string line in File.ReadLines(sourcePath)) { if (TypeOf(line) != "node") continue; string id = FieldValue(line, "id"); if (id != null && needed.Contains(id)) { w.WriteLine(line); result.Nodes++; } } foreach (var line in keep) w.WriteLine(line); } return result; } } public class AppInfo { public string Id; public string Name; public string Publisher; public string Version; public string Role; // own | dependency public bool HasSource; } public class AnalysisOptions { public string WorkspaceRoot; public int MaxListItems = 2000; /// <summary>callgraph.architecture.json; null or missing = parts derived from the folders.</summary> public string ArchitecturePath; } /// <summary>A loaded graph with its own apps, test apps and architecture: what the lookups and the scaffold work on.</summary> public class CallGraphWorkspace { public Graph Graph; public HashSet<string> Own; public HashSet<string> TestApps; public Architecture Architecture; public Explorer Explorer; public AppInfo[] Apps; public string WorkspaceRoot; } /// <summary>Metrics, findings, impact and the report model of a pruned graph.</summary> public static class Analyzer { static readonly Regex AttributeName = new Regex(@"^\[\s*([A-Za-z]+)", RegexOptions.Compiled); static bool IsMember(Node n) { return n.Kind == "Method" || n.Kind == "Trigger"; } /// <summary> /// Reads the attributes of own methods from the source, which the graph does not carry: [Test] marks a /// test, [EventSubscriber] a subscriber (platform events such as OnAfterModifyEvent have no publisher /// edge) and [ServiceEnabled] an action called from outside. The node line is the procedure line; the /// attributes and doc comments sit directly above it. /// </summary> static void MarkAttributes(Graph g, HashSet<string> own, Dictionary<string, string[]> files, HashSet<string> entryPoints) { foreach (var n in g.Nodes.Values) { if (n.Kind != "Method" || !own.Contains(n.App) || n.Line < 2) continue; var lines = FileLines(files, n.SourcePath); if (lines == null || n.Line > lines.Length) continue; for (int i = n.Line - 2; i >= 0 && i >= n.Line - 30; i--) { string text = lines[i].Trim(); if (text.Length == 0 || text.StartsWith("//", StringComparison.Ordinal)) continue; if (!text.StartsWith("[", StringComparison.Ordinal)) break; var m = AttributeName.Match(text); if (!m.Success) continue; switch (m.Groups[1].Value.ToLowerInvariant()) { case "test": n.Test = true; break; case "eventsubscriber": n.EventRole = "Subscriber"; entryPoints.Add(n.Id); break; case "serviceenabled": entryPoints.Add(n.Id); break; } } } } static string[] FileLines(Dictionary<string, string[]> cache, string path) { if (string.IsNullOrEmpty(path)) return null; string[] lines; if (cache.TryGetValue(path, out lines)) return lines; try { lines = File.ReadAllLines(path); } catch (IOException) { lines = null; } catch (UnauthorizedAccessException) { lines = null; } cache[path] = lines; return lines; } /// <summary> /// A hash of each own member's source text, from its line to the line before the next member of the /// same file. Lets a later build see a changed body even when the calls it makes did not change. /// </summary> static Dictionary<string, string> HashMembers(Graph g, HashSet<string> own, Dictionary<string, string[]> files) { var hashes = new Dictionary<string, string>(StringComparer.Ordinal); var byFile = g.Nodes.Values.Where(n => own.Contains(n.App) && IsMember(n) && n.Line > 0 && !string.IsNullOrEmpty(n.SourcePath)) .GroupBy(n => n.SourcePath, StringComparer.OrdinalIgnoreCase); using (var sha = SHA1.Create()) { foreach (var file in byFile) { var lines = FileLines(files, file.Key); if (lines == null) continue; var ordered = file.OrderBy(n => n.Line).ToList(); for (int i = 0; i < ordered.Count; i++) { int from = ordered[i].Line - 1; int to = i + 1 < ordered.Count ? ordered[i + 1].Line - 2 : lines.Length - 1; var sb = new StringBuilder(); for (int l = from; l <= to && l < lines.Length; l++) sb.Append(lines[l].Trim()).Append('\n'); byte[] h = sha.ComputeHash(Encoding.UTF8.GetBytes(sb.ToString())); hashes[ordered[i].Id] = BitConverter.ToString(h, 0, 8).Replace("-", "").ToLowerInvariant(); } } } return hashes; } static string Relative(string root, string path) { if (string.IsNullOrEmpty(root) || string.IsNullOrEmpty(path)) return path; string r = root.TrimEnd('\\', '/') + System.IO.Path.DirectorySeparatorChar; return path.StartsWith(r, StringComparison.OrdinalIgnoreCase) ? path.Substring(r.Length).Replace('\\', '/') : path; } static Dictionary<string, object> Obj(params object[] pairs) { var d = new Dictionary<string, object>(); for (int i = 0; i + 1 < pairs.Length; i += 2) d[(string)pairs[i]] = pairs[i + 1]; return d; } static string Label(Node n) { if (n == null) return ""; return string.IsNullOrEmpty(n.Member) ? n.ObjectType + " " + n.ObjectName : n.ObjectType + " " + n.ObjectName + "." + n.Member; } /// <summary> /// Loads a graph with its architecture: the file when it exists and parses, otherwise the folder-based parts /// (a broken file is reported as a warning, never thrown - reporting must not fail a build). /// </summary> public static CallGraphWorkspace Open(string graphPath, AppInfo[] apps, string workspaceRoot, string architecturePath) { var w = new CallGraphWorkspace(); w.Graph = Graph.Load(graphPath); w.Apps = apps ?? new AppInfo[0]; w.WorkspaceRoot = workspaceRoot; w.Own = new HashSet<string>(w.Apps.Where(a => a.Role == "own").Select(a => a.Id.ToLowerInvariant()), StringComparer.Ordinal); if (w.Own.Count == 0) foreach (var key in w.Graph.AppNames.Keys) w.Own.Add(key); var files = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase); MarkAttributes(w.Graph, w.Own, files, new HashSet<string>(StringComparer.Ordinal)); w.TestApps = new HashSet<string>(w.Graph.Nodes.Values.Where(n => n.Test).Select(n => n.App), StringComparer.Ordinal); w.Architecture = BuildArchitecture(w.Graph, w.Own, w.TestApps, w.Apps, workspaceRoot, architecturePath); w.Explorer = new Explorer(w.Graph, w.Architecture, w.Own); w.Architecture.Validate(w.Explorer.ObjectEdges()); return w; } static Architecture BuildArchitecture(Graph g, HashSet<string> own, HashSet<string> testApps, AppInfo[] apps, string workspaceRoot, string architecturePath) { var objects = Architecture.OwnObjects(g, own, workspaceRoot); Architecture arch = null; string problem = null; if (!string.IsNullOrEmpty(architecturePath) && File.Exists(architecturePath)) { try { arch = Architecture.Load(architecturePath); } catch (FormatException ex) { problem = ex.Message; } } if (arch == null) arch = Architecture.Fallback(objects, apps, testApps); if (problem != null) arch.Warnings.Add(problem + " The parts are derived from the folders instead."); arch.Assign(objects); // The source's /// summaries describe every object the file does not; the file overrides them. foreach (var kv in Architecture.SourceDescriptions(objects, workspaceRoot)) if (!arch.Descriptions.ContainsKey(kv.Key)) arch.Descriptions[kv.Key] = kv.Value; return arch; } public static Dictionary<string, object> Analyze(string graphPath, AppInfo[] apps, string baselineGraphPath, string baselineAnalysisPath, AnalysisOptions options) { if (options == null) options = new AnalysisOptions(); var g = Graph.Load(graphPath); var appList = apps ?? new AppInfo[0]; var own = new HashSet<string>(appList.Where(a => a.Role == "own").Select(a => a.Id.ToLowerInvariant()), StringComparer.Ordinal); if (own.Count == 0) foreach (var key in g.AppNames.Keys) own.Add(key); var files = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase); var entryPoints = new HashSet<string>(StringComparer.Ordinal); MarkAttributes(g, own, files, entryPoints); var testApps = new HashSet<string>(g.Nodes.Values.Where(n => n.Test).Select(n => n.App), StringComparer.Ordinal); var arch = BuildArchitecture(g, own, testApps, appList, options.WorkspaceRoot, options.ArchitecturePath); var explorer = new Explorer(g, arch, own); arch.Validate(explorer.ObjectEdges()); // Adjacency, ignoring the platform's built-in methods. var forward = new Dictionary<string, List<Edge>>(StringComparer.Ordinal); var backward = new Dictionary<string, List<Edge>>(StringComparer.Ordinal); foreach (var e in g.Edges) { if (Graph.AppOf(e.Callee) == Graph.BuiltInApp) continue; List<Edge> list; if (!forward.TryGetValue(e.Caller, out list)) forward[e.Caller] = list = new List<Edge>(); list.Add(e); if (!backward.TryGetValue(e.Callee, out list)) backward[e.Callee] = list = new List<Edge>(); list.Add(e); } Func<Node, bool> isOwnCode = n => own.Contains(n.App) && !testApps.Contains(n.App); var ownMembers = g.Nodes.Values.Where(n => own.Contains(n.App) && IsMember(n)).ToList(); var productMembers = ownMembers.Where(n => isOwnCode(n)).ToList(); // Reachability from the tests. var tests = ownMembers.Where(n => n.Test).Select(n => n.Id).ToList(); var reached = Walk(tests, forward, true); var untested = productMembers.Where(n => !reached.Contains(n.Id)).OrderBy(n => n.ObjectKey).ThenBy(n => n.Line).ToList(); // Calls of obsolete code (a method flagged obsolete, or any member of an obsolete object) from product // code. Tests touching obsolete objects (upgrade tests, say) are expected and not reported. var obsoleteCalls = new List<Dictionary<string, object>>(); foreach (var e in g.Edges) { Node caller, callee, calleeObject; if (!g.Nodes.TryGetValue(e.Caller, out caller) || !isOwnCode(caller) || caller.Obsolete) continue; if (!g.Nodes.TryGetValue(e.Callee, out callee)) continue; g.Nodes.TryGetValue(callee.ObjectKey, out calleeObject); Node callerObject; g.Nodes.TryGetValue(caller.ObjectKey, out callerObject); if (callerObject != null && callerObject.Obsolete) continue; if (callee.Obsolete || (calleeObject != null && calleeObject.Obsolete && calleeObject != caller && callee.ObjectKey != caller.ObjectKey)) { obsoleteCalls.Add(Obj("caller", Label(caller), "callee", Label(callee), "callerId", caller.Id, "calleeId", callee.Id, "path", Relative(options.WorkspaceRoot, e.SourcePath ?? caller.SourcePath), "line", e.Line > 0 ? e.Line : caller.Line, "dependency", !own.Contains(callee.App))); } } // Dead code: a non-public method of an own app that nothing calls. Two reachability paths the graph // does not model are honoured: interface dispatch, whose edges al graph resolves to one overload only // (so every overload of an interface-dispatched name counts), and calls by name - a notification // action or a Codeunit.Run target named in a string literal. var interfaceNames = new HashSet<string>(g.Edges.Where(e => e.Kind == "Interface") .Select(e => { Node t; return g.Nodes.TryGetValue(e.Callee, out t) ? t.ObjectKey + "#" + t.Member : null; }) .Where(k => k != null), StringComparer.OrdinalIgnoreCase); var ownFiles = ownMembers.Select(n => n.SourcePath).ToList(); var namesInLiterals = NamesInLiterals(ownFiles, files); // A call whose arguments use a type from a dependency without source is not bound by the // compiler, so its edge is missing. A name that is called anywhere in the text is not reported. var calledNames = CalledNames(ownFiles, files); // An interface from a dependency without source has no interface edges at all, so a codeunit that // implements one keeps every non-local method. var implementing = new HashSet<string>(g.Nodes.Values.Where(n => own.Contains(n.App) && n.Kind == "Object" && Implements(n, files)) .Select(n => n.ObjectKey), StringComparer.Ordinal); var dead = new List<Node>(); foreach (var n in ownMembers) { if (n.Kind != "Method" || n.Test || n.Obsolete || n.Access == "Public") continue; if (g.Subscribers.Contains(n.Id) || entryPoints.Contains(n.Id)) continue; if (interfaceNames.Contains(n.ObjectKey + "#" + n.Member)) continue; if (implementing.Contains(n.ObjectKey) && n.Access != "Local") continue; if (n.Member != null && (namesInLiterals.Contains(n.Member) || calledNames.Contains(n.Member))) continue; List<Edge> incoming; bool called = backward.TryGetValue(n.Id, out incoming) && incoming.Any(e => e.Caller != n.Id); if (!called) dead.Add(n); } dead = dead.OrderBy(n => n.ObjectKey).ThenBy(n => n.Line).ToList(); var hashes = HashMembers(g, own, files); var metrics = Metrics(g, own, testApps, ownMembers, productMembers, tests.Count, reached, untested.Count, obsoleteCalls.Count, dead.Count); var result = new Dictionary<string, object>(); result["schema"] = 1; result["generatedUtc"] = DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture); result["apps"] = appList.Select(a => (object)Obj("id", a.Id, "name", a.Name, "publisher", a.Publisher, "version", a.Version, "role", a.Role, "hasSource", a.HasSource, "testApp", testApps.Contains((a.Id ?? "").ToLowerInvariant()))).ToList(); result["metrics"] = metrics; int max = options.MaxListItems; result["findings"] = Obj( "untested", untested.Take(max).Select(n => (object)MemberRef(n, options)).ToList(), "untestedTotal", untested.Count, "obsoleteCalls", obsoleteCalls.Take(max).Cast<object>().ToList(), "obsoleteCallsTotal", obsoleteCalls.Count, "dead", dead.Take(max).Select(n => (object)MemberRef(n, options)).ToList(), "deadTotal", dead.Count); result["methodHashes"] = hashes; result["baselineMetrics"] = null; result["impact"] = null; if (!string.IsNullOrEmpty(baselineGraphPath) && File.Exists(baselineGraphPath)) { Dictionary<string, object> baselineAnalysis = null; if (!string.IsNullOrEmpty(baselineAnalysisPath) && File.Exists(baselineAnalysisPath)) baselineAnalysis = Json.Parse(File.ReadAllText(baselineAnalysisPath)) as Dictionary<string, object>; if (baselineAnalysis != null && baselineAnalysis.ContainsKey("metrics")) result["baselineMetrics"] = baselineAnalysis["metrics"]; result["impact"] = Impact(g, Graph.Load(baselineGraphPath), baselineAnalysis, own, testApps, hashes, forward, backward, options); } result["graph"] = ReportGraph(g, own, testApps, reached, tests.Count > 0, untested, dead, obsoleteCalls, options, arch); result["architecture"] = ArchitectureModel(arch); result["checks"] = Checks(g, own, testApps, ownMembers, options); metrics["unassignedObjects"] = arch.Objects.Count(o => !arch.Assignment.ContainsKey(o.Key)); metrics["describedObjects"] = arch.Objects.Count(o => arch.Descriptions.ContainsKey(o.Label)); metrics["architectureWarnings"] = arch.Warnings.Count; return result; } static Dictionary<string, object> ArchitectureModel(Architecture arch) { var index = new Dictionary<string, int>(StringComparer.Ordinal); for (int i = 0; i < arch.Parts.Count; i++) if (arch.Parts[i].Id != null && !index.ContainsKey(arch.Parts[i].Id)) index[arch.Parts[i].Id] = i; Func<string, object> idx = id => id != null && index.ContainsKey(id) ? (object)index[id] : null; return Obj( "source", arch.Source, "project", arch.ProjectName, "summary", arch.ProjectSummary, "parts", arch.Parts.Select(p => (object)Obj("id", p.Id, "label", p.Label, "kind", p.Kind, "parent", idx(p.ParentId), "summary", p.Summary)).ToList(), "flows", arch.Flows.Select(f => (object)Obj("id", f.Id, "title", f.Title, "summary", f.Summary, "steps", f.Steps.Select(s => (object)Obj("node", s.Node, "title", s.Title, "note", s.Note)).ToList())).ToList(), "tour", arch.Tour.Select(s => (object)Obj("node", s.Node, "title", s.Title, "note", s.Note)).ToList(), "warnings", arch.Warnings.Cast<object>().ToList(), "objects", arch.Objects.Count, "unassigned", arch.Objects.Count(o => !arch.Assignment.ContainsKey(o.Key)), "described", arch.Objects.Count(o => arch.Descriptions.ContainsKey(o.Label))); } /// <summary> /// AL checks from the node attributes al graph carries. Measured on Banking: OnPrem scope appears only in /// Microsoft code and is reached only through Microsoft's own calls, so a "cloud reaches OnPrem" check would /// report nothing but false alarms and is left out on purpose. What is reported is information, not a defect. /// </summary> static Dictionary<string, object> Checks(Graph g, HashSet<string> own, HashSet<string> testApps, List<Node> ownMembers, AnalysisOptions options) { Func<Node, bool> elevated = n => n != null && !string.IsNullOrEmpty(n.Elevation) && n.Elevation != "None"; var callersOf = new Dictionary<string, HashSet<string>>(StringComparer.Ordinal); foreach (var e in g.Edges) { if (!own.Contains(Graph.AppOf(e.Caller))) continue; string target = Graph.ObjectKeyOf(e.Callee), source = Graph.ObjectKeyOf(e.Caller); if (target == source) continue; HashSet<string> set; if (!callersOf.TryGetValue(target, out set)) callersOf[target] = set = new HashSet<string>(StringComparer.Ordinal); set.Add(source); } Func<Node, object> objectRow = n => { HashSet<string> callers; callersOf.TryGetValue(n.ObjectKey, out callers); return Obj("object", n.ObjectType + " " + n.ObjectName, "elevation", n.Elevation, "app", n.App, "path", Relative(options.WorkspaceRoot, n.SourcePath), "line", n.Line, "calledFrom", callers == null ? new List<object>() : callers.Select(c => { Node o; return (object)(g.Nodes.TryGetValue(c, out o) ? o.ObjectType + " " + o.ObjectName : c); }).OrderBy(x => x).Take(50).ToList()); }; var objectNodes = g.Nodes.Values.Where(n => n.Kind == "Object" || n.Kind == "Table").ToList(); var elevatedOwn = objectNodes.Where(n => own.Contains(n.App) && elevated(n)).OrderBy(n => n.ObjectKey).Select(objectRow).ToList(); var elevatedDependency = objectNodes.Where(n => !own.Contains(n.App) && n.App != Graph.BuiltInApp && elevated(n) && callersOf.ContainsKey(n.ObjectKey)) .OrderBy(n => n.ObjectKey).Select(objectRow).ToList(); var nonDebuggable = ownMembers.Where(n => n.NonDebuggable && !testApps.Contains(n.App)).OrderBy(n => n.ObjectKey).ThenBy(n => n.Line).ToList(); return Obj( "elevatedOwn", elevatedOwn, "elevatedDependency", elevatedDependency, "nonDebuggable", nonDebuggable.Take(options.MaxListItems).Select(n => (object)MemberRef(n, options)).ToList(), "nonDebuggableTotal", nonDebuggable.Count, "notChecked", "Cloud code reaching OnPrem-scoped members: on measured apps OnPrem scope occurs only in Microsoft code and is reached only through Microsoft's own calls, which SaaS allows; a direct call is a compile error anyway."); } static readonly Regex ImplementsClause = new Regex(@"\bimplements\b", RegexOptions.IgnoreCase | RegexOptions.Compiled); /// <summary>Whether the object declaration (up to its opening brace) has an implements clause.</summary> static bool Implements(Node objectNode, Dictionary<string, string[]> files) { var lines = FileLines(files, objectNode.SourcePath); if (lines == null || objectNode.Line < 1) return false; for (int i = objectNode.Line - 1; i < lines.Length && i < objectNode.Line + 8; i++) { string code = lines[i]; int comment = code.IndexOf("//", StringComparison.Ordinal); if (comment >= 0) code = code.Substring(0, comment); int brace = code.IndexOf('{'); if (ImplementsClause.IsMatch(brace >= 0 ? code.Substring(0, brace) : code)) return true; if (brace >= 0) return false; } return false; } static readonly Regex IdentifierLiteral = new Regex(@"'([A-Za-z_][A-Za-z0-9_]*)'", RegexOptions.Compiled); static HashSet<string> NamesInLiterals(IEnumerable<string> paths, Dictionary<string, string[]> files) { var names = new HashSet<string>(StringComparer.OrdinalIgnoreCase); foreach (var path in paths.Where(p => !string.IsNullOrEmpty(p)).Distinct(StringComparer.OrdinalIgnoreCase)) { var lines = FileLines(files, path); if (lines == null) continue; foreach (var line in lines) { if (line.IndexOf('\'') < 0) continue; foreach (Match m in IdentifierLiteral.Matches(line)) names.Add(m.Groups[1].Value); } } return names; } static readonly Regex CallSite = new Regex(@"(?<![A-Za-z0-9_""])([A-Za-z_][A-Za-z0-9_]*)\s*\(", RegexOptions.Compiled); static readonly Regex Declaration = new Regex(@"^\s*((local|internal|protected)\s+)?(procedure|trigger|event)\b", RegexOptions.IgnoreCase | RegexOptions.Compiled); /// <summary>Identifiers followed by '(' outside procedure/trigger declarations and comments.</summary> static HashSet<string> CalledNames(IEnumerable<string> paths, Dictionary<string, string[]> files) { var names = new HashSet<string>(StringComparer.OrdinalIgnoreCase); foreach (var path in paths.Where(p => !string.IsNullOrEmpty(p)).Distinct(StringComparer.OrdinalIgnoreCase)) { var lines = FileLines(files, path); if (lines == null) continue; foreach (var raw in lines) { string line = raw; int comment = line.IndexOf("//", StringComparison.Ordinal); if (comment >= 0) line = line.Substring(0, comment); if (line.IndexOf('(') < 0 || Declaration.IsMatch(line)) continue; foreach (Match m in CallSite.Matches(line)) names.Add(m.Groups[1].Value); } } return names; } static HashSet<string> Walk(IEnumerable<string> start, Dictionary<string, List<Edge>> adjacency, bool forwardDirection) { var seen = new HashSet<string>(StringComparer.Ordinal); var queue = new Queue<string>(); foreach (var s in start) if (seen.Add(s)) queue.Enqueue(s); while (queue.Count > 0) { string id = queue.Dequeue(); List<Edge> edges; if (!adjacency.TryGetValue(id, out edges)) continue; foreach (var e in edges) { string next = forwardDirection ? e.Callee : e.Caller; if (seen.Add(next)) queue.Enqueue(next); } } return seen; } static Dictionary<string, object> MemberRef(Node n, AnalysisOptions o) { return Obj("id", n.Id, "object", n.ObjectType + " " + n.ObjectName, "member", n.Member, "kind", n.Kind, "path", Relative(o.WorkspaceRoot, n.SourcePath), "line", n.Line); } static Dictionary<string, object> Metrics(Graph g, HashSet<string> own, HashSet<string> testApps, List<Node> ownMembers, List<Node> productMembers, int testCount, HashSet<string> reached, int untestedCount, int obsoleteCalls, int deadCount) { var edgesByKind = new SortedDictionary<string, object>(StringComparer.Ordinal); int edges = 0, dependencyCalls = 0, platformCalls = 0; var dependencyCallees = new HashSet<string>(StringComparer.Ordinal); foreach (var e in g.Edges) { if (!own.Contains(Graph.AppOf(e.Caller))) continue; string calleeApp = Graph.AppOf(e.Callee); if (calleeApp == Graph.BuiltInApp) { platformCalls++; continue; } edges++; object count; edgesByKind[e.Kind ?? "?"] = edgesByKind.TryGetValue(e.Kind ?? "?", out count) ? (object)((int)count + 1) : 1; if (!own.Contains(calleeApp)) { dependencyCalls++; dependencyCallees.Add(e.Callee); } } int reachedCount = productMembers.Count(n => reached.Contains(n.Id)); var m = new Dictionary<string, object>(); m["objects"] = g.Nodes.Values.Count(n => own.Contains(n.App) && (n.Kind == "Object" || n.Kind == "Table")); m["methods"] = ownMembers.Count(n => n.Kind == "Method"); m["triggers"] = ownMembers.Count(n => n.Kind == "Trigger"); m["edges"] = edges; m["edgesByKind"] = edgesByKind; m["platformCalls"] = platformCalls; m["dependencyCalls"] = dependencyCalls; m["dependencyMembersCalled"] = dependencyCallees.Count; // A stitched graph carries events and interfaces as edges (publisher -> subscriber, interface // method -> implementation); the shard records for them are gone by then. var publishers = new HashSet<string>(g.Nodes.Values.Where(n => own.Contains(n.App) && n.EventRole == "Publisher").Select(n => n.Id), StringComparer.Ordinal); publishers.UnionWith(g.Publishers.Where(p => own.Contains(Graph.AppOf(p)))); var subscribers = new HashSet<string>(g.Edges.Where(e => e.Kind == "Event" && own.Contains(Graph.AppOf(e.Callee))).Select(e => e.Callee), StringComparer.Ordinal); subscribers.UnionWith(g.Subscribers.Where(s => own.Contains(Graph.AppOf(s)))); subscribers.UnionWith(g.Nodes.Values.Where(n => own.Contains(n.App) && n.EventRole == "Subscriber").Select(n => n.Id)); var implementations = new HashSet<string>(g.Edges.Where(e => e.Kind == "Interface" && own.Contains(Graph.AppOf(e.Callee))).Select(e => e.Callee), StringComparer.Ordinal); m["eventPublishers"] = publishers.Count; m["eventSubscribers"] = subscribers.Count; m["interfaceImplementations"] = implementations.Count; m["testMethods"] = testCount; m["productMembers"] = productMembers.Count; m["reachedByTests"] = testCount > 0 ? (object)reachedCount : null; m["notReachedByTests"] = testCount > 0 ? (object)untestedCount : null; m["notReachedPercent"] = testCount > 0 && productMembers.Count > 0 ? (object)Math.Round(100.0 * untestedCount / productMembers.Count, 1) : null; m["obsoleteCalls"] = obsoleteCalls; m["deadMethods"] = deadCount; return m; } static Dictionary<string, object> Impact(Graph current, Graph baseline, Dictionary<string, object> baselineAnalysis, HashSet<string> own, HashSet<string> testApps, Dictionary<string, string> hashes, Dictionary<string, List<Edge>> forward, Dictionary<string, List<Edge>> backward, AnalysisOptions options) { var baseHashes = new Dictionary<string, string>(StringComparer.Ordinal); if (baselineAnalysis != null && baselineAnalysis.ContainsKey("methodHashes")) { var h = baselineAnalysis["methodHashes"] as Dictionary<string, object>; if (h != null) foreach (var kv in h) baseHashes[kv.Key] = Convert.ToString(kv.Value, CultureInfo.InvariantCulture); } var now = current.Nodes.Values.Where(n => own.Contains(n.App) && IsMember(n)).ToDictionary(n => n.Id, StringComparer.Ordinal); var before = baseline.Nodes.Values.Where(n => own.Contains(n.App) && IsMember(n)).ToDictionary(n => n.Id, StringComparer.Ordinal); var baseCalls = baseline.Edges.GroupBy(e => e.Caller, StringComparer.Ordinal) .ToDictionary(x => x.Key, x => string.Join("|", x.Select(e => e.Kind + ">" + e.Callee).OrderBy(s => s, StringComparer.Ordinal)), StringComparer.Ordinal); var nowCalls = current.Edges.GroupBy(e => e.Caller, StringComparer.Ordinal) .ToDictionary(x => x.Key, x => string.Join("|", x.Select(e => e.Kind + ">" + e.Callee).OrderBy(s => s, StringComparer.Ordinal)), StringComparer.Ordinal); var added = now.Keys.Where(k => !before.ContainsKey(k)).ToList(); var removed = before.Keys.Where(k => !now.ContainsKey(k)).ToList(); var modified = new List<string>(); foreach (var id in now.Keys.Where(k => before.ContainsKey(k))) { string a, b, ha, hb; nowCalls.TryGetValue(id, out a); baseCalls.TryGetValue(id, out b); bool callsChanged = (a ?? "") != (b ?? ""); bool bodyChanged = hashes.TryGetValue(id, out ha) && baseHashes.TryGetValue(id, out hb) && ha != hb; if (callsChanged || bodyChanged) modified.Add(id); } // Who is affected: everything that (transitively) calls a changed member, plus the former callers of // a removed one that still exist. var seeds = new List<string>(added); seeds.AddRange(modified); var removedSet = new HashSet<string>(removed, StringComparer.Ordinal); foreach (var e in baseline.Edges) if (removedSet.Contains(e.Callee) && now.ContainsKey(e.Caller)) seeds.Add(e.Caller); var callers = Walk(seeds, backward, false); var changedSet = new HashSet<string>(added.Concat(modified), StringComparer.Ordinal); var affected = new List<Node>(); var affectedTests = new List<Node>(); foreach (var id in callers) { Node n; if (!current.Nodes.TryGetValue(id, out n) || !own.Contains(n.App) || !IsMember(n)) continue; if (n.Test) affectedTests.Add(n); else if (!changedSet.Contains(id) && !testApps.Contains(n.App)) affected.Add(n); } int max = options.MaxListItems; Func<IEnumerable<string>, Dictionary<string, Node>, List<object>> refs = (ids, map) => ids.Select(id => map[id]).OrderBy(n => n.ObjectKey).ThenBy(n => n.Line).Take(max).Select(n => (object)MemberRef(n, options)).ToList(); var affectedObjects = affected.Concat(changedSet.Select(id => now[id])).Select(n => n.ObjectKey).Distinct().ToList(); return Obj( "added", refs(added, now), "addedTotal", added.Count, "removed", refs(removed, before), "removedTotal", removed.Count, "modified", refs(modified, now), "modifiedTotal", modified.Count, "affected", affected.OrderBy(n => n.ObjectKey).ThenBy(n => n.Line).Take(max).Select(n => (object)MemberRef(n, options)).ToList(), "affectedTotal", affected.Count, "affectedObjects", affectedObjects.Count, "tests", affectedTests.OrderBy(n => n.ObjectKey).ThenBy(n => n.Line).Take(max).Select(n => (object)MemberRef(n, options)).ToList(), "testsTotal", affectedTests.Count, "changedIds", changedSet.ToList(), "affectedIds", affected.Select(n => n.Id).ToList(), "bodyHashesCompared", baseHashes.Count > 0); } /// <summary>The object-level model the HTML report renders, with the members of each object for drill-down.</summary> static Dictionary<string, object> ReportGraph(Graph g, HashSet<string> own, HashSet<string> testApps, HashSet<string> reached, bool haveTests, List<Node> untested, List<Node> dead, List<Dictionary<string, object>> obsoleteCalls, AnalysisOptions options, Architecture arch) { var partIndex = new Dictionary<string, int>(StringComparer.Ordinal); for (int i = 0; i < arch.Parts.Count; i++) if (arch.Parts[i].Id != null && !partIndex.ContainsKey(arch.Parts[i].Id)) partIndex[arch.Parts[i].Id] = i; // Where a user starts: pages, reports, queries and xmlports, and their extensions. var entryTypes = new HashSet<string>(new[] { "Page", "PageExtension", "Report", "ReportExtension", "Query", "XmlPort" }, StringComparer.OrdinalIgnoreCase); var untestedSet = new HashSet<string>(untested.Select(n => n.Id), StringComparer.Ordinal); var deadSet = new HashSet<string>(dead.Select(n => n.Id), StringComparer.Ordinal); var obsoleteCallers = new HashSet<string>(obsoleteCalls.Select(c => (string)c["callerId"]), StringComparer.Ordinal); var objectIndex = new Dictionary<string, int>(StringComparer.Ordinal); var objects = new List<object>(); var members = new List<object>(); var memberIndex = new Dictionary<string, int>(StringComparer.Ordinal); Func<string, int> objectOf = key => { int idx; if (objectIndex.TryGetValue(key, out idx)) return idx; Node o; g.Nodes.TryGetValue(key, out o); string app = Graph.AppOf(key); idx = objects.Count; objectIndex[key] = idx; string type = Graph.ObjectTypeOf(key); string rest = key.Substring(key.IndexOf(" / ", StringComparison.Ordinal) + 3 + type.Length).Trim(); int space = rest.IndexOf(' '); string name = space > 0 ? rest.Substring(space + 1) : rest; string partId, description; arch.Assignment.TryGetValue(key, out partId); arch.Descriptions.TryGetValue(type + " " + name, out description); objects.Add(Obj("k", key, "type", type, "id", space > 0 ? rest.Substring(0, space) : "0", "name", name, "app", app, "own", own.Contains(app), "test", testApps.Contains(app), "obsolete", o != null && o.Obsolete, "path", o != null ? Relative(options.WorkspaceRoot, o.SourcePath) : null, "line", o != null ? o.Line : 0, "members", new List<object>(), "part", partId != null && partIndex.ContainsKey(partId) ? (object)partIndex[partId] : null, "desc", description, "entry", own.Contains(app) && !testApps.Contains(app) && entryTypes.Contains(type), "elev", o != null && o.Elevation != null && o.Elevation != "None" ? o.Elevation : null)); return idx; }; foreach (var n in g.Nodes.Values.Where(x => x.App != Graph.BuiltInApp).OrderBy(x => x.ObjectKey).ThenBy(x => x.Line)) { if (n.Kind == "Object" || n.Kind == "Table") { objectOf(n.ObjectKey); continue; } int obj = objectOf(n.ObjectKey); int idx = members.Count; memberIndex[n.Id] = idx; var flags = new List<object>(); if (n.Test) flags.Add("test"); if (n.Obsolete) flags.Add("obsolete"); if (haveTests && untestedSet.Contains(n.Id)) flags.Add("untested"); if (deadSet.Contains(n.Id)) flags.Add("dead"); if (obsoleteCallers.Contains(n.Id)) flags.Add("callsObsolete"); if (g.Subscribers.Contains(n.Id)) flags.Add("subscriber"); if (n.EventRole == "Publisher") flags.Add("publisher"); if (n.EventRole == "Subscriber" && !flags.Contains("subscriber")) flags.Add("subscriber"); if (n.NonDebuggable) flags.Add("nonDebuggable"); members.Add(Obj("k", n.Id, "o", obj, "n", n.Member ?? n.Kind, "kind", n.Kind, "access", n.Access, "line", n.Line, "flags", flags)); ((List<object>)((Dictionary<string, object>)objects[obj])["members"]).Add(idx); } var calls = new List<object>(); var objectEdges = new Dictionary<string, Dictionary<string, object>>(StringComparer.Ordinal); foreach (var e in g.Edges) { int a, b; if (!memberIndex.TryGetValue(e.Caller, out a) || !memberIndex.TryGetValue(e.Callee, out b)) continue; calls.Add(new object[] { a, b, e.Kind }); int oa = (int)((Dictionary<string, object>)members[a])["o"]; int ob = (int)((Dictionary<string, object>)members[b])["o"]; if (oa == ob) continue; string key = oa + ">" + ob; Dictionary<string, object> oe; if (!objectEdges.TryGetValue(key, out oe)) { oe = Obj("s", oa, "t", ob, "n", 0, "kinds", new List<object>()); objectEdges[key] = oe; } oe["n"] = (int)oe["n"] + 1; var kinds = (List<object>)oe["kinds"]; if (!kinds.Contains(e.Kind)) kinds.Add(e.Kind); } return Obj("objects", objects, "members", members, "calls", calls, "objectEdges", objectEdges.Values.Cast<object>().ToList(), "appNames", g.AppNames); } } } |