// Copyright Keysight Technologies 2012-2019 // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, you can obtain one at http://mozilla.org/MPL/2.0/. using System; using System.Collections.Generic; using System.ComponentModel; using System.IO; using System.Linq; using System.Text; namespace OpenTap { /// Class for generating the summary for a test plan. [Browsable(false)] class TestPlanRunSummaryListener : ResultListener, IArtifactListener { private int stepRunLimit => EngineSettings.Current.SummaryTestStepLimit; // Avoid holding on to references to TestStepRuns (fat objects) and only store the information // we need. struct TestStepRunData { public readonly Guid Parent; public readonly Guid Id; public readonly Verdict Verdict; public readonly string TestStepName; public readonly TimeSpan Duration; TestStepRunData(TestStepRun run) { Parent = run.Parent; Verdict = run.Verdict; Id = run.Id; TestStepName = run.TestStepName; Duration = run.Duration; } public static TestStepRunData FromTestStepRun(TestStepRun run) { return new TestStepRunData(run); } } // null when stepRunLimit is exceeded. // parent run to child run. Dictionary stepRuns = new Dictionary(); TestPlanRun planRun; static readonly TraceSource summaryLog = OpenTap.Log.CreateSource("Summary"); /// Creates an instance and removes the ResultListener log from the TraceSources, so log messages from this listener wont go anywhere. public TestPlanRunSummaryListener() { Name = "Summary"; OpenTap.Log.RemoveSource(Log); } /// Clears the memory. /// public override void OnTestPlanRunStart(TestPlanRun planRun) { stepRuns = new Dictionary(); this.planRun = planRun; foreach (var f in tempFiles) { f.Dispose(); } tempFiles.Clear(); artifacts.Clear(); } /// Saves which steps are started. /// public override void OnTestStepRunStart(TestStepRun stepRun) { if (stepRun == null) throw new ArgumentNullException(nameof(stepRun)); base.OnTestStepRunStart(stepRun); if (stepRuns == null) return; stepRuns[stepRun.Id] = TestStepRunData.FromTestStepRun(stepRun); if (stepRuns.Count > stepRunLimit) { stepRuns = null; } } public override void OnTestStepRunCompleted(TestStepRun stepRun) { // this is mostly for updating stepRun's duration. if (stepRun == null) throw new ArgumentNullException(nameof(stepRun)); base.OnTestStepRunCompleted(stepRun); if (stepRuns == null) return; stepRuns[stepRun.Id] = TestStepRunData.FromTestStepRun(stepRun); if (stepRuns.Count > stepRunLimit) { // Too many steps, so we skip the summary. stepRuns = null; } } static int stepRunIndent(IDictionary stepRuns, TestStepRunData stepRun) { int indent = 0; TestStepRunData p = stepRun; while (stepRuns.ContainsKey(p.Parent)) { p = stepRuns[p.Parent]; indent++; } return indent; } static void printSummary(IDictionary stepRuns, TestStepRunData run, int maxIndent, int idx, ILookup lookup) { int indentcnt = stepRunIndent(stepRuns, run); string indent = new String(' ', indentcnt * 2); string inverseIndent = new String(' ', maxIndent * 2 - indentcnt * 2); string v = run.Verdict == Verdict.NotSet ? "" : run.Verdict.ToString(); var name = run.TestStepName; // this prints something like this: '12:42:16.302 Summary Delay 106 ms' summaryLog.Info($"{indent} {name,-43} {inverseIndent}{ShortTimeSpan.LongTimeSpanFormat(run.Duration)} {v,-7}"); int idx2 = 0; foreach (TestStepRunData run2 in lookup[run.Id]) printSummary(stepRuns, run2, maxIndent, idx2, lookup); } private readonly string separator = new string('-', 44); static string FormatSize(long size) { if (size < 1000) return $"{size} B"; if (size < 1000000) return $"{Math.Round(((double)size) / 1000, 1)} kB"; return $"{Math.Round(((double)size) / 1000_000, 1)} MB"; } /// Prints the artifact summary. public void PrintArtifactsSummary() { string formatSummary(string message) { int fillLength = (int)Math.Floor((maxLength - message.Length) / 2.0); StringBuilder sb = new StringBuilder(new string('-', fillLength)); sb.Append(message); sb.Append('-', maxLength - sb.Length); return sb.ToString(); } if (artifacts.Count > 0) { summaryLog.Info(formatSummary($" {artifacts.Count} artifacts registered. ")); foreach (var artifact in artifacts) { var artifactSize = new FileInfo(artifact).Length; summaryLog.Info($" - {Path.GetFullPath(artifact)} [{FormatSize(artifactSize)}]"); } } else { //summaryLog.Info(formatSummary($" No artifacts registered. ")); } } int maxLength = -1; /// Prints the summary. public void PrintSummary() { Dictionary thisRuns = null; Utils.Swap(ref thisRuns, ref stepRuns); maxLength = -1; if (planRun == null) return; //Something very wrong happened. In this case the use will be informed of an error anyway. bool hasOtherVerdict = planRun.Verdict != Verdict.Pass && planRun.Verdict != Verdict.NotSet; ILookup parentLookup = null; int maxIndent = 0; if (thisRuns != null) { parentLookup = thisRuns.Values.ToLookup(v => v.Parent); Func getMaxIndent = null; getMaxIndent = guid => 1 + parentLookup[guid].Select(step => getMaxIndent(step.Id)).DefaultIfEmpty(0).Max(); maxIndent = getMaxIndent(planRun.Id); } int maxVerdictLength = hasOtherVerdict ? planRun.Verdict.ToString().Length : 0; if (thisRuns != null) { foreach(TestStepRunData run in parentLookup[planRun.Id]) { int max = getMaxVerdictLength(run, maxVerdictLength, parentLookup); if (max > maxVerdictLength) { maxVerdictLength = max; } } } int addPadLength = maxIndent + (int)Math.Round(maxVerdictLength / 2.0); Func formatSummaryHeader = (message, indentLength) => { string indent = new String('-', indentLength + 3); StringBuilder sb = new StringBuilder(indent); sb.Append(message); sb.Append(indent); maxLength = sb.Length; return sb.ToString(); }; Func formatSummary = (message) => { int fillLength = (int)Math.Floor((maxLength - message.Length) / 2.0); StringBuilder sb = new StringBuilder(new string('-', fillLength)); sb.Append(message); sb.Append('-', maxLength - sb.Length); return sb.ToString(); }; summaryLog.Info(formatSummaryHeader($" Summary of test plan started {planRun.StartTime} ", addPadLength)); if (thisRuns != null) { int idx = 0; foreach (TestStepRunData run in parentLookup[planRun.Id]) printSummary(thisRuns, run, maxIndent, idx, parentLookup); } else { summaryLog.Warning("Test plan summary skipped. Summary contains too many steps ({0}).", stepRunLimit); } summaryLog.Info(formatSummary(separator)); if (!hasOtherVerdict) { summaryLog.Info(formatSummary($" Test plan completed successfully in {ShortTimeSpan.LongTimeSpanFormat(planRun.Duration)} ")); } else { summaryLog.Info(formatSummary(string.Format(" Test plan completed with verdict {1} in {0,6} ", ShortTimeSpan.LongTimeSpanFormat(planRun.Duration), planRun.Verdict))); } } int getMaxVerdictLength(TestStepRunData run, int maxVerdictLength, ILookup lookup) { int maxLength = run.Verdict != Verdict.NotSet ? run.Verdict.ToString().Length : 0; foreach(TestStepRunData run2 in lookup[run.Id]) { int max = getMaxVerdictLength(run2, maxLength, lookup); if (max > maxLength) { maxLength = max; } } return maxLength > maxVerdictLength ? maxLength : maxVerdictLength; } // keeps track of published artifacts. HashSet artifacts { get; set; } = new HashSet(); // these tmp files are used to keep temporary (stream) files alive until the next test plan run. readonly List tempFiles = new List(); readonly string targetLoc = Path.Combine(Path.GetTempPath(), "OpenTAP", "Temporary Artifacts", Guid.NewGuid().ToString()); public void OnArtifactPublished(TestRun run, Stream artifactStream, string artifactName) { // when an artifact is published, keep track of it. // if it is a file artifact (FileStream), we just keep track of the names. // if is a non-artifact stream, persist it on disk for a while. using var _ = artifactStream; if (artifactStream is FileStream originFileStream) { artifacts.Add(originFileStream.Name); return; } var tmpFileName = Path.Combine(targetLoc, Path.GetFileName(artifactName)); if (artifacts.Contains(tmpFileName)) return; FileSystemHelper.EnsureDirectoryOf(tmpFileName); // DeleteOnClose: persist the file until it is closed (next test plan run). var fileStream = new FileStream(tmpFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose); artifactStream.CopyTo(fileStream); fileStream.Flush(); artifacts.Add(tmpFileName); tempFiles.Add(fileStream); } } }