// Copyright Keysight Technologies 2012-2019
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, you can obtain one at http://mozilla.org/MPL/2.0/.
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using System.Text;
namespace OpenTap
{
/// Class for generating the summary for a test plan.
[Browsable(false)]
class TestPlanRunSummaryListener : ResultListener, IArtifactListener
{
private int stepRunLimit => EngineSettings.Current.SummaryTestStepLimit;
// Avoid holding on to references to TestStepRuns (fat objects) and only store the information
// we need.
struct TestStepRunData
{
public readonly Guid Parent;
public readonly Guid Id;
public readonly Verdict Verdict;
public readonly string TestStepName;
public readonly TimeSpan Duration;
TestStepRunData(TestStepRun run)
{
Parent = run.Parent;
Verdict = run.Verdict;
Id = run.Id;
TestStepName = run.TestStepName;
Duration = run.Duration;
}
public static TestStepRunData FromTestStepRun(TestStepRun run)
{
return new TestStepRunData(run);
}
}
// null when stepRunLimit is exceeded.
// parent run to child run.
Dictionary stepRuns = new Dictionary();
TestPlanRun planRun;
static readonly TraceSource summaryLog = OpenTap.Log.CreateSource("Summary");
/// Creates an instance and removes the ResultListener log from the TraceSources, so log messages from this listener wont go anywhere.
public TestPlanRunSummaryListener()
{
Name = "Summary";
OpenTap.Log.RemoveSource(Log);
}
/// Clears the memory.
///
public override void OnTestPlanRunStart(TestPlanRun planRun)
{
stepRuns = new Dictionary();
this.planRun = planRun;
foreach (var f in tempFiles)
{
f.Dispose();
}
tempFiles.Clear();
artifacts.Clear();
}
/// Saves which steps are started.
///
public override void OnTestStepRunStart(TestStepRun stepRun)
{
if (stepRun == null)
throw new ArgumentNullException(nameof(stepRun));
base.OnTestStepRunStart(stepRun);
if (stepRuns == null) return;
stepRuns[stepRun.Id] = TestStepRunData.FromTestStepRun(stepRun);
if (stepRuns.Count > stepRunLimit)
{
stepRuns = null;
}
}
public override void OnTestStepRunCompleted(TestStepRun stepRun)
{ // this is mostly for updating stepRun's duration.
if (stepRun == null)
throw new ArgumentNullException(nameof(stepRun));
base.OnTestStepRunCompleted(stepRun);
if (stepRuns == null)
return;
stepRuns[stepRun.Id] = TestStepRunData.FromTestStepRun(stepRun);
if (stepRuns.Count > stepRunLimit)
{
// Too many steps, so we skip the summary.
stepRuns = null;
}
}
static int stepRunIndent(IDictionary stepRuns, TestStepRunData stepRun)
{
int indent = 0;
TestStepRunData p = stepRun;
while (stepRuns.ContainsKey(p.Parent))
{
p = stepRuns[p.Parent];
indent++;
}
return indent;
}
static void printSummary(IDictionary stepRuns, TestStepRunData run, int maxIndent, int idx, ILookup lookup)
{
int indentcnt = stepRunIndent(stepRuns, run);
string indent = new String(' ', indentcnt * 2);
string inverseIndent = new String(' ', maxIndent * 2 - indentcnt * 2);
string v = run.Verdict == Verdict.NotSet ? "" : run.Verdict.ToString();
var name = run.TestStepName;
// this prints something like this: '12:42:16.302 Summary Delay 106 ms'
summaryLog.Info($"{indent} {name,-43} {inverseIndent}{ShortTimeSpan.LongTimeSpanFormat(run.Duration)} {v,-7}");
int idx2 = 0;
foreach (TestStepRunData run2 in lookup[run.Id])
printSummary(stepRuns, run2, maxIndent, idx2, lookup);
}
private readonly string separator = new string('-', 44);
static string FormatSize(long size)
{
if (size < 1000)
return $"{size} B";
if (size < 1000000)
return $"{Math.Round(((double)size) / 1000, 1)} kB";
return $"{Math.Round(((double)size) / 1000_000, 1)} MB";
}
/// Prints the artifact summary.
public void PrintArtifactsSummary()
{
string formatSummary(string message)
{
int fillLength = (int)Math.Floor((maxLength - message.Length) / 2.0);
StringBuilder sb = new StringBuilder(new string('-', fillLength));
sb.Append(message);
sb.Append('-', maxLength - sb.Length);
return sb.ToString();
}
if (artifacts.Count > 0)
{
summaryLog.Info(formatSummary($" {artifacts.Count} artifacts registered. "));
foreach (var artifact in artifacts)
{
var artifactSize = new FileInfo(artifact).Length;
summaryLog.Info($" - {Path.GetFullPath(artifact)} [{FormatSize(artifactSize)}]");
}
}
else
{
//summaryLog.Info(formatSummary($" No artifacts registered. "));
}
}
int maxLength = -1;
/// Prints the summary.
public void PrintSummary()
{
Dictionary thisRuns = null;
Utils.Swap(ref thisRuns, ref stepRuns);
maxLength = -1;
if (planRun == null) return; //Something very wrong happened. In this case the use will be informed of an error anyway.
bool hasOtherVerdict = planRun.Verdict != Verdict.Pass && planRun.Verdict != Verdict.NotSet;
ILookup parentLookup = null;
int maxIndent = 0;
if (thisRuns != null)
{
parentLookup = thisRuns.Values.ToLookup(v => v.Parent);
Func getMaxIndent = null;
getMaxIndent = guid => 1 + parentLookup[guid].Select(step => getMaxIndent(step.Id)).DefaultIfEmpty(0).Max();
maxIndent = getMaxIndent(planRun.Id);
}
int maxVerdictLength = hasOtherVerdict ? planRun.Verdict.ToString().Length : 0;
if (thisRuns != null)
{
foreach(TestStepRunData run in parentLookup[planRun.Id])
{
int max = getMaxVerdictLength(run, maxVerdictLength, parentLookup);
if (max > maxVerdictLength)
{
maxVerdictLength = max;
}
}
}
int addPadLength = maxIndent + (int)Math.Round(maxVerdictLength / 2.0);
Func formatSummaryHeader = (message, indentLength) => {
string indent = new String('-', indentLength + 3);
StringBuilder sb = new StringBuilder(indent);
sb.Append(message);
sb.Append(indent);
maxLength = sb.Length;
return sb.ToString();
};
Func formatSummary = (message) => {
int fillLength = (int)Math.Floor((maxLength - message.Length) / 2.0);
StringBuilder sb = new StringBuilder(new string('-', fillLength));
sb.Append(message);
sb.Append('-', maxLength - sb.Length);
return sb.ToString();
};
summaryLog.Info(formatSummaryHeader($" Summary of test plan started {planRun.StartTime} ", addPadLength));
if (thisRuns != null)
{
int idx = 0;
foreach (TestStepRunData run in parentLookup[planRun.Id])
printSummary(thisRuns, run, maxIndent, idx, parentLookup);
}
else
{
summaryLog.Warning("Test plan summary skipped. Summary contains too many steps ({0}).", stepRunLimit);
}
summaryLog.Info(formatSummary(separator));
if (!hasOtherVerdict)
{
summaryLog.Info(formatSummary($" Test plan completed successfully in {ShortTimeSpan.LongTimeSpanFormat(planRun.Duration)} "));
}
else
{
summaryLog.Info(formatSummary(string.Format(" Test plan completed with verdict {1} in {0,6} ", ShortTimeSpan.LongTimeSpanFormat(planRun.Duration), planRun.Verdict)));
}
}
int getMaxVerdictLength(TestStepRunData run, int maxVerdictLength, ILookup lookup)
{
int maxLength = run.Verdict != Verdict.NotSet ? run.Verdict.ToString().Length : 0;
foreach(TestStepRunData run2 in lookup[run.Id])
{
int max = getMaxVerdictLength(run2, maxLength, lookup);
if (max > maxLength)
{
maxLength = max;
}
}
return maxLength > maxVerdictLength ? maxLength : maxVerdictLength;
}
// keeps track of published artifacts.
HashSet artifacts { get; set; } = new HashSet();
// these tmp files are used to keep temporary (stream) files alive until the next test plan run.
readonly List tempFiles = new List();
readonly string targetLoc = Path.Combine(Path.GetTempPath(), "OpenTAP", "Temporary Artifacts", Guid.NewGuid().ToString());
public void OnArtifactPublished(TestRun run, Stream artifactStream, string artifactName)
{
// when an artifact is published, keep track of it.
// if it is a file artifact (FileStream), we just keep track of the names.
// if is a non-artifact stream, persist it on disk for a while.
using var _ = artifactStream;
if (artifactStream is FileStream originFileStream)
{
artifacts.Add(originFileStream.Name);
return;
}
var tmpFileName = Path.Combine(targetLoc, Path.GetFileName(artifactName));
if (artifacts.Contains(tmpFileName)) return;
FileSystemHelper.EnsureDirectoryOf(tmpFileName);
// DeleteOnClose: persist the file until it is closed (next test plan run).
var fileStream = new FileStream(tmpFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, 4096, FileOptions.DeleteOnClose);
artifactStream.CopyTo(fileStream);
fileStream.Flush();
artifacts.Add(tmpFileName);
tempFiles.Add(fileStream);
}
}
}