// Copyright Keysight Technologies 2012-2019 // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, you can obtain one at http://mozilla.org/MPL/2.0/. using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; using System.IO; using System.Linq; using System.Runtime.Serialization; using System.Threading; using System.Threading.Tasks; using System.Xml.Serialization; namespace OpenTap { /// /// Common base class for and . /// [Serializable] [DataContract(IsReference = true)] [KnownType(typeof(TestPlanRun))] [KnownType(typeof(TestStepRun))] public abstract class TestRun { /// /// ID of this test run; can be used to uniquely identify a or . /// [DataMember] public Guid Id { get; protected set; } /// Creates a new TestRun public TestRun() { Id = Guid.NewGuid(); Parameters = new ResultParameters(); } internal const string GROUP = ""; int verdictIndex = -1; /// /// resulting from the run. /// [DataMember] [MetaData(macroName: "Verdict")] public Verdict Verdict { get { switch ( Parameters.GetIndexed((nameof(Verdict), GROUP), ref verdictIndex)) { case Verdict verdict: return verdict; case string r: return (Verdict) StringConvertProvider.FromString(r, TypeData.FromType(typeof(Verdict)), null); default: return Verdict.NotSet; // unexpected, but let's not fail. } } protected internal set => Parameters.SetIndexed((nameof(Verdict), GROUP), ref verdictIndex, value); } /// Exception causing the Verdict to be 'Error'. public Exception Exception { get; set; } /// Length of time it took to run. public virtual TimeSpan Duration { get { var param = Parameters[nameof(Duration), GROUP]; if (param is double duration) return Time.FromSeconds(duration); return TimeSpan.Zero; } internal protected set => Parameters[nameof(Duration), GROUP] = value.TotalSeconds; } /// /// Time when the test started as a object. /// [DataMember] [MetaData(macroName: "Date")] public DateTime StartTime { get { var param = Parameters[nameof(StartTime), GROUP]; if (param is DateTime startTime) { return startTime; } return new DateTime(); } set => Parameters[nameof(StartTime), GROUP] = value; } /// /// Time when the test started as ticks of the high resolution hardware counter. /// Use with and to convert to a timestamp. /// [DataMember] public long StartTimeStamp { get; protected set; } ResultParameters parameters; /// /// A list of parameters associated with this run that can be used by . /// [DataMember] public ResultParameters Parameters { get => parameters; protected set { if (ReferenceEquals(parameters, value)) return; parameters = value; verdictIndex = -1; } } /// /// Upgrades . /// /// internal void UpgradeVerdict(Verdict verdict) { // locks are slow. // Hence first check if a verdict upgrade is needed, then do the actual upgrade. if (Verdict < verdict) { lock (upgradeVerdictLock) { if (Verdict < verdict) Verdict = verdict; } } } internal readonly object upgradeVerdictLock = new object(); /// Calculated abort condition. internal BreakCondition BreakCondition { get; set; } /// This is invoked when a child run is started. /// internal virtual void ChildStarted(TestStepRun stepRun) { } } /// /// Test step Run parameters. /// Contains information about a test step run. Unique for each time a test plan is run. /// If the same step is run multiple times during the same TestStep run, multiple instances of this object will be created. /// [Serializable] [DataContract(IsReference = true)] [KnownType(typeof(TestPlanRun))] [DebuggerDisplay("TestStepRun {TestStepName}")] public class TestStepRun : TestRun { readonly TestPlanRun testPlanRun; /// /// Parent run that is above this run in the tree. /// [DataMember] public Guid Parent { get; private set; } /// /// of the that this run represents. /// [DataMember] public Guid TestStepId { get; protected set; } /// /// of the that this run represents. /// [DataMember] public string TestStepName { get; protected set; } /// /// Assembly qualified name of the type of that this run represents. /// [DataMember] public string TestStepTypeName { get; protected set; } /// /// If possible, the next step executed. This can be implemented to support 'jump-to' functionality. /// It requires that the suggested next step ID belongs to one of the sibling steps of TestStepId. /// [XmlIgnore] public Guid? SuggestedNextStep { get; set; } /// /// True if the step currently supports jumping to a step other than the next. /// Only true while the step is running or at BreakOffered. /// [XmlIgnore] public bool SupportsJumpTo { get; set; } /// This step run was skipped without execution. internal bool Skipped { get; set; } /// /// The path name of the steps. E.g the name of the step combined with all of its parent steps. /// internal string TestStepPath; readonly ManualResetEventSlim completedEvent = new ManualResetEventSlim(false); static readonly CancellationToken NoToken = CancellationToken.None; /// Waits for the test step run to be entirely done. This includes any deferred processing. public void WaitForCompletion() { WaitForCompletion(NoToken); } /// Waits for the test step run to be entirely done. This includes any deferred processing. It does not break when the test plan is aborted public void WaitForCompletion(CancellationToken cancellationToken) { if (completedEvent.IsSet) return; var currentThread = TapThread.Current; if(!WasDeferred && StepThread == currentThread) throw new InvalidOperationException("StepRun.WaitForCompletion called from the thread itself. This will either cause a deadlock or do nothing."); if (cancellationToken == NoToken) { completedEvent.Wait(); return; } var waits = new[] { completedEvent.WaitHandle, cancellationToken.WaitHandle }; while (WaitHandle.WaitAny(waits, 100) == WaitHandle.WaitTimeout) { if (completedEvent.Wait(0)) break; } } /// The thread in which the step is running. public TapThread StepThread { get; private set; } /// Set to true if the step execution has been deferred. internal bool WasDeferred => (this.ResultSource as ResultSource)?.WasDeferred ?? false; #region Internal Members used by the TestPlan /// Called by TestStep.DoRun before running the step. internal void StartStepRun() { if (Verdict != Verdict.NotSet) throw new ArgumentOutOfRangeException(nameof(Verdict), "StepRun.StartStepRun has already been called once."); StepThread = TapThread.Current; StartTime = DateTime.Now; StartTimeStamp = Stopwatch.GetTimestamp(); } /// Called by TestStep.DoRun after running the step. internal void CompleteStepRun(TestPlanRun planRun, ITestStep step, TimeSpan runDuration) { // update values in the run. ResultParameters.UpdateParams(Parameters, step); Duration = runDuration; // Requires update after TestStepRunStart and before TestStepRunCompleted UpgradeVerdict(step.Verdict); } internal void SignalCompleted() { StepThread = null; completedEvent.Set(); } ITypeData stepTypeData; private ITestStep _step; TestStepRun(ITestStep step) { _step = step; TestStepId = step.Id; TestStepName = step.GetFormattedName(); stepTypeData = TypeData.GetTypeData(step); TestStepTypeName = stepTypeData.AsTypeData().AssemblyQualifiedName; Parameters = ResultParameters.GetParams(step, stepTypeData); Verdict = Verdict.NotSet; } internal void UpdateParams() { ResultParameters.UpdateParams(Parameters, _step, type: stepTypeData); } /// /// Constructor for TestStepRun. /// /// Property Step. /// Property Parent. /// Parameters that will be stored together with the actual parameters of the steps. public TestStepRun(ITestStep step, Guid parent, IEnumerable attachedParameters = null): this(step) { if (attachedParameters != null) Parameters.AddRange(attachedParameters); Parent = parent; } internal TestStepRun(ITestStep step, TestRun parent, IEnumerable attachedParameters, TestPlanRun testPlanRun): this(step) { this.testPlanRun = testPlanRun; if (attachedParameters != null) Parameters.AddRange(attachedParameters); Parent = parent.Id; BreakCondition = calculateBreakCondition(step, parent); } static BreakCondition calculateBreakCondition(ITestStep step, TestRun parentStepRun) { BreakCondition breakCondition = BreakConditionProperty.GetBreakCondition(step); if (breakCondition.HasFlag(BreakCondition.Inherit)) return parentStepRun.BreakCondition | BreakCondition.Inherit; return breakCondition; } internal TestStepRun Clone() { var run = (TestStepRun) this.MemberwiseClone(); run.Parameters = run.Parameters.Clone(); return run; } #endregion /// Returns true if the break conditions are satisfied for the test step run. public bool BreakConditionsSatisfied() { var verdict = Verdict; if (OutOfRetries || (verdict == Verdict.Fail && BreakCondition.HasFlag(BreakCondition.BreakOnFail)) || (verdict == Verdict.Error && BreakCondition.HasFlag(BreakCondition.BreakOnError)) || (verdict == Verdict.Inconclusive && BreakCondition.HasFlag(BreakCondition.BreakOnInconclusive)) || (verdict == Verdict.Pass && BreakCondition.HasFlag(BreakCondition.BreakOnPass))) { return true; } return false; } internal bool OutOfRetries { get; set; } internal bool Completed => deferDone.IsSet; internal void ThrowDueToBreakConditions() { throw new TestStepBreakException(_step, this); } internal bool IsStepChildOf(ITestStep step, ITestStep possibleParent) { do { step = step.Parent as ITestStep; } while (step != null && step != possibleParent); return step != null; } internal void WaitForOutput(OutputAvailability mode, ITestStep waitingFor) { ITestStep currentStep = TestStepExtensions.currentlyExecutingTestStep; var currentThread = TapThread.Current; switch (mode) { case OutputAvailability.BeforeRun: return; case OutputAvailability.AfterDefer: if ((StepThread == currentThread && runDone.Wait(0) == false) || (currentStep != null && IsStepChildOf(currentStep, waitingFor))) throw new Exception("Deadlock detected"); deferDone.Wait(TapThread.Current.AbortToken); break; case OutputAvailability.AfterRun: if ((StepThread == currentThread && runDone.Wait(0) == false) || (currentStep != null && IsStepChildOf(currentStep, waitingFor))) throw new Exception("Deadlock detected"); runDone.Wait(TapThread.Current.AbortToken); break; } } ManualResetEventSlim runDone = new ManualResetEventSlim(false, 0); ManualResetEventSlim deferDone = new ManualResetEventSlim(false, 0); internal void AfterRun(ITestStep step) { // load member data for results. List resultMembers = null; List primitiveMembers = null; foreach (var member in stepTypeData.GetMembers()) { if (member.HasAttribute()) { var td = member.TypeDescriptor.AsTypeData(); if (td != null && (td.IsPrimitive() || td.IsString)) { primitiveMembers ??= new List(); primitiveMembers.Add(member); } else { resultMembers ??= new List(); resultMembers.Add(member); } } } void publishResults() { // primitive members are collapsed into one row with several columns with the step // name as a result name, and each (primitive) property as a column. if (primitiveMembers != null) { var arrays = primitiveMembers.SelectValues(r => { var value = r.GetValue(step); // skip null values. if (value == null) return null; var array = Array.CreateInstance(value.GetType(), 1); array.SetValue(value, 0); return array; }).ToArray(); var names = primitiveMembers.Select(r => r.GetDisplayAttribute().Name).ToList(); ((ResultSource)ResultSource).PublishTable(step.StepRun.TestStepName, names, arrays); } if (resultMembers != null) { // handle the non-primitive members normally. foreach (var r in resultMembers) { if (r.TypeDescriptor.IsPrimitive()) continue; var value = r.GetValue(step); // skip null values. if (value == null) continue; var name = r.GetDisplayAttribute().Name; ((ResultSource)ResultSource).Publish(name, value); } } } // ResultSource may be null. if (ResultSource != null && (resultMembers != null || primitiveMembers != null)) { if (WasDeferred) ResultSource.Defer(publishResults); else publishResults(); } runDone.Set(); if (WasDeferred && ResultSource != null) { ResultSource.Defer(() => { deferDone.Set(); stepRuns = ImmutableDictionary.Empty; }); } else { deferDone.Set(); stepRuns = ImmutableDictionary.Empty; } } internal IResultSource ResultSource; /// Sets the result source for this run. public void SetResultSource(IResultSource resultSource) => this.ResultSource = resultSource; bool isCompleted => completedEvent.IsSet; List<(Guid, Guid)> waitingFor = new List<(Guid, Guid)>(); /// Will throw an exception when it times out. internal TestStepRun WaitForChildStepStart(Guid childStep, bool wait, Guid waiterStep) { if (stepRuns.TryGetValue(childStep, out var run)) return run; if (!wait) return null; if (isCompleted) return null; var sem = new ManualResetEventSlim(false, 0 ); TestStepRun stepRun = null; void onChildStarted(TestStepRun id) { if (childStep == id.TestStepId) { sem.Set(); stepRun = id; } } lock (waitingFor) { childStarted += onChildStarted; waitingFor.Add((waiterStep, childStep)); if (Utils2.IsLooped(waitingFor, waiterStep)) throw new InvalidOperationException("Input / output loop detected in WaitForChildStepStart"); } try { while (true) { if (stepRuns.TryGetValue(childStep, out run)) return run; if (sem.Wait(5000, TapThread.Current.AbortToken)) return stepRun; } } finally { lock (waitingFor) { childStarted -= onChildStarted; waitingFor.Remove((waiterStep, childStep)); } } } Action childStarted; // we keep a mapping of the most recent run of any child step. This is important to be able to update inputs. // the guid is the ID of a step. ImmutableDictionary stepRuns = ImmutableDictionary.Empty; internal override void ChildStarted(TestStepRun stepRun) { base.ChildStarted(stepRun); Utils.InterlockedSwap(ref stepRuns, () => stepRuns.SetItem(stepRun.TestStepId, stepRun)); childStarted?.Invoke(stepRun); } /// Publishes an artifact for the test plan run. /// The artifact data as a stream. When publishing an artifact stream, the stream will be disposed by the callee and does not have to be disposed by the caller. /// The name of the published artifact. public void PublishArtifact(Stream stream, string artifactName) => testPlanRun.PublishArtifactWithRunAsync(stream, artifactName, this).Wait(); /// Publishes an artifact for the test plan run. /// The artifact data as a stream. When publishing an artifact stream, the stream will be disposed by the callee and does not have to be disposed by the caller. /// The name of the published artifact. public Task PublishArtifactAsync(Stream stream, string artifactName) => testPlanRun.PublishArtifactWithRunAsync(stream, artifactName, this); /// Publishes an artifact file for the test plan run. public void PublishArtifact(string file) => testPlanRun.PublishArtifactWithRunAsync(file, this).Wait(); /// Publishes an artifact file for the test plan run. public Task PublishArtifactAsync(string file) => testPlanRun.PublishArtifactWithRunAsync(file, this); } class TestStepBreakException : OperationCanceledException { public string TestStepName => Step.GetFormattedName(); public Verdict Verdict => Run.Verdict; public ITestStep Step { get; set; } public TestStepRun Run { get; set; } public TestStepBreakException(ITestStep step, TestStepRun run) { Step = step; Run = run; } public override string Message => $"Break issued from '{TestStepName}' due to verdict {Verdict}. See Break Conditions settings."; } }