// Copyright Keysight Technologies 2012-2019
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, you can obtain one at http://mozilla.org/MPL/2.0/.
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.Serialization;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Serialization;
namespace OpenTap
{
///
/// Common base class for and .
///
[Serializable]
[DataContract(IsReference = true)]
[KnownType(typeof(TestPlanRun))]
[KnownType(typeof(TestStepRun))]
public abstract class TestRun
{
///
/// ID of this test run; can be used to uniquely identify a or .
///
[DataMember]
public Guid Id { get; protected set; }
/// Creates a new TestRun
public TestRun()
{
Id = Guid.NewGuid();
Parameters = new ResultParameters();
}
internal const string GROUP = "";
int verdictIndex = -1;
///
/// resulting from the run.
///
[DataMember]
[MetaData(macroName: "Verdict")]
public Verdict Verdict
{
get
{
switch ( Parameters.GetIndexed((nameof(Verdict), GROUP), ref verdictIndex))
{
case Verdict verdict:
return verdict;
case string r:
return (Verdict) StringConvertProvider.FromString(r, TypeData.FromType(typeof(Verdict)), null);
default:
return Verdict.NotSet; // unexpected, but let's not fail.
}
}
protected internal set => Parameters.SetIndexed((nameof(Verdict), GROUP), ref verdictIndex, value);
}
/// Exception causing the Verdict to be 'Error'.
public Exception Exception { get; set; }
/// Length of time it took to run.
public virtual TimeSpan Duration
{
get
{
var param = Parameters[nameof(Duration), GROUP];
if (param is double duration)
return Time.FromSeconds(duration);
return TimeSpan.Zero;
}
internal protected set => Parameters[nameof(Duration), GROUP] = value.TotalSeconds;
}
///
/// Time when the test started as a object.
///
[DataMember]
[MetaData(macroName: "Date")]
public DateTime StartTime {
get
{
var param = Parameters[nameof(StartTime), GROUP];
if (param is DateTime startTime)
{
return startTime;
}
return new DateTime();
}
set => Parameters[nameof(StartTime), GROUP] = value;
}
///
/// Time when the test started as ticks of the high resolution hardware counter.
/// Use with and to convert to a timestamp.
///
[DataMember]
public long StartTimeStamp { get; protected set; }
ResultParameters parameters;
///
/// A list of parameters associated with this run that can be used by .
///
[DataMember]
public ResultParameters Parameters
{
get => parameters;
protected set
{
if (ReferenceEquals(parameters, value)) return;
parameters = value;
verdictIndex = -1;
}
}
///
/// Upgrades .
///
///
internal void UpgradeVerdict(Verdict verdict)
{
// locks are slow.
// Hence first check if a verdict upgrade is needed, then do the actual upgrade.
if (Verdict < verdict)
{
lock (upgradeVerdictLock)
{
if (Verdict < verdict)
Verdict = verdict;
}
}
}
internal readonly object upgradeVerdictLock = new object();
/// Calculated abort condition.
internal BreakCondition BreakCondition { get; set; }
/// This is invoked when a child run is started.
///
internal virtual void ChildStarted(TestStepRun stepRun)
{
}
}
///
/// Test step Run parameters.
/// Contains information about a test step run. Unique for each time a test plan is run.
/// If the same step is run multiple times during the same TestStep run, multiple instances of this object will be created.
///
[Serializable]
[DataContract(IsReference = true)]
[KnownType(typeof(TestPlanRun))]
[DebuggerDisplay("TestStepRun {TestStepName}")]
public class TestStepRun : TestRun
{
readonly TestPlanRun testPlanRun;
///
/// Parent run that is above this run in the tree.
///
[DataMember]
public Guid Parent { get; private set; }
///
/// of the that this run represents.
///
[DataMember]
public Guid TestStepId { get; protected set; }
///
/// of the that this run represents.
///
[DataMember]
public string TestStepName { get; protected set; }
///
/// Assembly qualified name of the type of that this run represents.
///
[DataMember]
public string TestStepTypeName { get; protected set; }
///
/// If possible, the next step executed. This can be implemented to support 'jump-to' functionality.
/// It requires that the suggested next step ID belongs to one of the sibling steps of TestStepId.
///
[XmlIgnore]
public Guid? SuggestedNextStep { get; set; }
///
/// True if the step currently supports jumping to a step other than the next.
/// Only true while the step is running or at BreakOffered.
///
[XmlIgnore]
public bool SupportsJumpTo { get; set; }
/// This step run was skipped without execution.
internal bool Skipped { get; set; }
///
/// The path name of the steps. E.g the name of the step combined with all of its parent steps.
///
internal string TestStepPath;
readonly ManualResetEventSlim completedEvent = new ManualResetEventSlim(false);
static readonly CancellationToken NoToken = CancellationToken.None;
/// Waits for the test step run to be entirely done. This includes any deferred processing.
public void WaitForCompletion()
{
WaitForCompletion(NoToken);
}
/// Waits for the test step run to be entirely done. This includes any deferred processing. It does not break when the test plan is aborted
public void WaitForCompletion(CancellationToken cancellationToken)
{
if (completedEvent.IsSet) return;
var currentThread = TapThread.Current;
if(!WasDeferred && StepThread == currentThread) throw new InvalidOperationException("StepRun.WaitForCompletion called from the thread itself. This will either cause a deadlock or do nothing.");
if (cancellationToken == NoToken)
{
completedEvent.Wait();
return;
}
var waits = new[] { completedEvent.WaitHandle, cancellationToken.WaitHandle };
while (WaitHandle.WaitAny(waits, 100) == WaitHandle.WaitTimeout)
{
if (completedEvent.Wait(0))
break;
}
}
/// The thread in which the step is running.
public TapThread StepThread { get; private set; }
/// Set to true if the step execution has been deferred.
internal bool WasDeferred => (this.ResultSource as ResultSource)?.WasDeferred ?? false;
#region Internal Members used by the TestPlan
/// Called by TestStep.DoRun before running the step.
internal void StartStepRun()
{
if (Verdict != Verdict.NotSet)
throw new ArgumentOutOfRangeException(nameof(Verdict), "StepRun.StartStepRun has already been called once.");
StepThread = TapThread.Current;
StartTime = DateTime.Now;
StartTimeStamp = Stopwatch.GetTimestamp();
}
/// Called by TestStep.DoRun after running the step.
internal void CompleteStepRun(TestPlanRun planRun, ITestStep step, TimeSpan runDuration)
{
// update values in the run.
ResultParameters.UpdateParams(Parameters, step);
Duration = runDuration; // Requires update after TestStepRunStart and before TestStepRunCompleted
UpgradeVerdict(step.Verdict);
}
internal void SignalCompleted()
{
StepThread = null;
completedEvent.Set();
}
ITypeData stepTypeData;
private ITestStep _step;
TestStepRun(ITestStep step)
{
_step = step;
TestStepId = step.Id;
TestStepName = step.GetFormattedName();
stepTypeData = TypeData.GetTypeData(step);
TestStepTypeName = stepTypeData.AsTypeData().AssemblyQualifiedName;
Parameters = ResultParameters.GetParams(step, stepTypeData);
Verdict = Verdict.NotSet;
}
internal void UpdateParams()
{
ResultParameters.UpdateParams(Parameters, _step, type: stepTypeData);
}
///
/// Constructor for TestStepRun.
///
/// Property Step.
/// Property Parent.
/// Parameters that will be stored together with the actual parameters of the steps.
public TestStepRun(ITestStep step, Guid parent, IEnumerable attachedParameters = null): this(step)
{
if (attachedParameters != null) Parameters.AddRange(attachedParameters);
Parent = parent;
}
internal TestStepRun(ITestStep step, TestRun parent, IEnumerable attachedParameters, TestPlanRun testPlanRun): this(step)
{
this.testPlanRun = testPlanRun;
if (attachedParameters != null) Parameters.AddRange(attachedParameters);
Parent = parent.Id;
BreakCondition = calculateBreakCondition(step, parent);
}
static BreakCondition calculateBreakCondition(ITestStep step, TestRun parentStepRun)
{
BreakCondition breakCondition = BreakConditionProperty.GetBreakCondition(step);
if (breakCondition.HasFlag(BreakCondition.Inherit))
return parentStepRun.BreakCondition | BreakCondition.Inherit;
return breakCondition;
}
internal TestStepRun Clone()
{
var run = (TestStepRun) this.MemberwiseClone();
run.Parameters = run.Parameters.Clone();
return run;
}
#endregion
/// Returns true if the break conditions are satisfied for the test step run.
public bool BreakConditionsSatisfied()
{
var verdict = Verdict;
if (OutOfRetries
|| (verdict == Verdict.Fail && BreakCondition.HasFlag(BreakCondition.BreakOnFail))
|| (verdict == Verdict.Error && BreakCondition.HasFlag(BreakCondition.BreakOnError))
|| (verdict == Verdict.Inconclusive && BreakCondition.HasFlag(BreakCondition.BreakOnInconclusive))
|| (verdict == Verdict.Pass && BreakCondition.HasFlag(BreakCondition.BreakOnPass)))
{
return true;
}
return false;
}
internal bool OutOfRetries { get; set; }
internal bool Completed => deferDone.IsSet;
internal void ThrowDueToBreakConditions()
{
throw new TestStepBreakException(_step, this);
}
internal bool IsStepChildOf(ITestStep step, ITestStep possibleParent)
{
do
{
step = step.Parent as ITestStep;
}
while (step != null && step != possibleParent);
return step != null;
}
internal void WaitForOutput(OutputAvailability mode, ITestStep waitingFor)
{
ITestStep currentStep = TestStepExtensions.currentlyExecutingTestStep;
var currentThread = TapThread.Current;
switch (mode)
{
case OutputAvailability.BeforeRun:
return;
case OutputAvailability.AfterDefer:
if ((StepThread == currentThread && runDone.Wait(0) == false) ||
(currentStep != null && IsStepChildOf(currentStep, waitingFor)))
throw new Exception("Deadlock detected");
deferDone.Wait(TapThread.Current.AbortToken);
break;
case OutputAvailability.AfterRun:
if ((StepThread == currentThread && runDone.Wait(0) == false) ||
(currentStep != null && IsStepChildOf(currentStep, waitingFor)))
throw new Exception("Deadlock detected");
runDone.Wait(TapThread.Current.AbortToken);
break;
}
}
ManualResetEventSlim runDone = new ManualResetEventSlim(false, 0);
ManualResetEventSlim deferDone = new ManualResetEventSlim(false, 0);
internal void AfterRun(ITestStep step)
{
// load member data for results.
List resultMembers = null;
List primitiveMembers = null;
foreach (var member in stepTypeData.GetMembers())
{
if (member.HasAttribute())
{
var td = member.TypeDescriptor.AsTypeData();
if (td != null && (td.IsPrimitive() || td.IsString))
{
primitiveMembers ??= new List();
primitiveMembers.Add(member);
}
else
{
resultMembers ??= new List();
resultMembers.Add(member);
}
}
}
void publishResults()
{
// primitive members are collapsed into one row with several columns with the step
// name as a result name, and each (primitive) property as a column.
if (primitiveMembers != null)
{
var arrays = primitiveMembers.SelectValues(r =>
{
var value = r.GetValue(step);
// skip null values.
if (value == null)
return null;
var array = Array.CreateInstance(value.GetType(), 1);
array.SetValue(value, 0);
return array;
}).ToArray();
var names = primitiveMembers.Select(r => r.GetDisplayAttribute().Name).ToList();
((ResultSource)ResultSource).PublishTable(step.StepRun.TestStepName, names, arrays);
}
if (resultMembers != null)
{
// handle the non-primitive members normally.
foreach (var r in resultMembers)
{
if (r.TypeDescriptor.IsPrimitive())
continue;
var value = r.GetValue(step);
// skip null values.
if (value == null)
continue;
var name = r.GetDisplayAttribute().Name;
((ResultSource)ResultSource).Publish(name, value);
}
}
}
// ResultSource may be null.
if (ResultSource != null && (resultMembers != null || primitiveMembers != null))
{
if (WasDeferred)
ResultSource.Defer(publishResults);
else
publishResults();
}
runDone.Set();
if (WasDeferred && ResultSource != null)
{
ResultSource.Defer(() =>
{
deferDone.Set();
stepRuns = ImmutableDictionary.Empty;
});
}
else
{
deferDone.Set();
stepRuns = ImmutableDictionary.Empty;
}
}
internal IResultSource ResultSource;
/// Sets the result source for this run.
public void SetResultSource(IResultSource resultSource) => this.ResultSource = resultSource;
bool isCompleted => completedEvent.IsSet;
List<(Guid, Guid)> waitingFor = new List<(Guid, Guid)>();
/// Will throw an exception when it times out.
internal TestStepRun WaitForChildStepStart(Guid childStep, bool wait, Guid waiterStep)
{
if (stepRuns.TryGetValue(childStep, out var run))
return run;
if (!wait) return null;
if (isCompleted) return null;
var sem = new ManualResetEventSlim(false, 0 );
TestStepRun stepRun = null;
void onChildStarted(TestStepRun id)
{
if (childStep == id.TestStepId)
{
sem.Set();
stepRun = id;
}
}
lock (waitingFor)
{
childStarted += onChildStarted;
waitingFor.Add((waiterStep, childStep));
if (Utils2.IsLooped(waitingFor, waiterStep))
throw new InvalidOperationException("Input / output loop detected in WaitForChildStepStart");
}
try
{
while (true)
{
if (stepRuns.TryGetValue(childStep, out run))
return run;
if (sem.Wait(5000, TapThread.Current.AbortToken))
return stepRun;
}
}
finally
{
lock (waitingFor)
{
childStarted -= onChildStarted;
waitingFor.Remove((waiterStep, childStep));
}
}
}
Action childStarted;
// we keep a mapping of the most recent run of any child step. This is important to be able to update inputs.
// the guid is the ID of a step.
ImmutableDictionary stepRuns = ImmutableDictionary.Empty;
internal override void ChildStarted(TestStepRun stepRun)
{
base.ChildStarted(stepRun);
Utils.InterlockedSwap(ref stepRuns, () => stepRuns.SetItem(stepRun.TestStepId, stepRun));
childStarted?.Invoke(stepRun);
}
/// Publishes an artifact for the test plan run.
/// The artifact data as a stream. When publishing an artifact stream, the stream will be disposed by the callee and does not have to be disposed by the caller.
/// The name of the published artifact.
public void PublishArtifact(Stream stream, string artifactName)
=> testPlanRun.PublishArtifactWithRunAsync(stream, artifactName, this).Wait();
/// Publishes an artifact for the test plan run.
/// The artifact data as a stream. When publishing an artifact stream, the stream will be disposed by the callee and does not have to be disposed by the caller.
/// The name of the published artifact.
public Task PublishArtifactAsync(Stream stream, string artifactName)
=> testPlanRun.PublishArtifactWithRunAsync(stream, artifactName, this);
/// Publishes an artifact file for the test plan run.
public void PublishArtifact(string file) => testPlanRun.PublishArtifactWithRunAsync(file, this).Wait();
/// Publishes an artifact file for the test plan run.
public Task PublishArtifactAsync(string file) => testPlanRun.PublishArtifactWithRunAsync(file, this);
}
class TestStepBreakException : OperationCanceledException
{
public string TestStepName => Step.GetFormattedName();
public Verdict Verdict => Run.Verdict;
public ITestStep Step { get; set; }
public TestStepRun Run { get; set; }
public TestStepBreakException(ITestStep step, TestStepRun run)
{
Step = step;
Run = run;
}
public override string Message =>
$"Break issued from '{TestStepName}' due to verdict {Verdict}. See Break Conditions settings.";
}
}