using System; using System.Linq; using System.Threading; using NUnit.Framework; using OpenTap.Engine.UnitTests.TestTestSteps; using OpenTap.Plugins.BasicSteps; namespace OpenTap.Engine.UnitTests { [TestFixture] public class BreakConditionsTest { public class TestStepFailsTimes : TestStep { public int fails = 0; public int timesRun = 0; public override void PrePlanRun() { fails = 5; timesRun = 0; base.PrePlanRun(); } public override void Run() { timesRun += 1; if (fails > 0) { fails -= 1; throw new Exception("Intended failure!"); } UpgradeVerdict(Verdict.Pass); } } // The new abort condition system gives many possibilities. // 1. everything inherits (same as without) // 2. step not allowed to fail // 3. and or step not allowed to error // 4. step allowed to fail // 5. and or step allowed to error // Retry: // 2. Single step retry until not fail. // - Step passes // - Step creates an error // - Step never passes -> break? // 3. Parent step retry until child steps pass // what about inconclusive?? [TestCase(Verdict.Error, BreakCondition.BreakOnError)] [TestCase(Verdict.Fail, BreakCondition.BreakOnFail)] [TestCase(Verdict.Inconclusive, BreakCondition.BreakOnInconclusive)] [TestCase(Verdict.Pass, BreakCondition.BreakOnPass)] [TestCase(Verdict.Inconclusive, BreakCondition.BreakOnInconclusive | BreakCondition.BreakOnError)] public void TestStepBreakOnError(Verdict verdictOutput, object _condition) { // _condition arg cannot be a BreakCondition as BreakCondition is not public. BreakCondition condition = (BreakCondition) _condition; var l = new PlanRunCollectorListener(); TestPlan plan = new TestPlan(); var verdict = new VerdictStep { VerdictOutput = verdictOutput }; BreakConditionProperty.SetBreakCondition(plan, condition); var verdict2 = new VerdictStep { VerdictOutput = Verdict.Pass }; plan.Steps.Add(verdict); plan.Steps.Add(verdict2); var run = plan.Execute(new[] {l}); Assert.AreEqual(verdictOutput, run.Verdict); Assert.AreEqual(1, l.StepRuns.Count); Assert.AreEqual(BreakCondition.Inherit, BreakConditionProperty.GetBreakCondition(verdict2)); var log = l.LogString; Assert.IsTrue(log.Contains("Break issued from")); } [TestCase(Verdict.Pass, EngineSettings.AbortTestPlanType.Step_Error, 2)] [TestCase(Verdict.Fail, EngineSettings.AbortTestPlanType.Step_Error, 2)] [TestCase(Verdict.Error, EngineSettings.AbortTestPlanType.Step_Error, 1)] [TestCase(Verdict.Fail, EngineSettings.AbortTestPlanType.Step_Fail, 1)] [TestCase(Verdict.Pass, EngineSettings.AbortTestPlanType.Step_Pass, 1)] [TestCase(Verdict.Inconclusive, EngineSettings.AbortTestPlanType.Step_Inconclusive, 1)] [TestCase(Verdict.Fail, EngineSettings.AbortTestPlanType.Step_Error | EngineSettings.AbortTestPlanType.Step_Fail, 1)] [TestCase(Verdict.Error, EngineSettings.AbortTestPlanType.Step_Error | EngineSettings.AbortTestPlanType.Step_Fail, 1)] [TestCase(Verdict.Pass, EngineSettings.AbortTestPlanType.Step_Error | EngineSettings.AbortTestPlanType.Step_Fail, 2)] public void EngineInheritedConditions(Verdict verdictOutput, EngineSettings.AbortTestPlanType abortTestPlanType, int runCount) { Verdict finalVerdict = verdictOutput; var prev = EngineSettings.Current.AbortTestPlan; try { EngineSettings.Current.AbortTestPlan = abortTestPlanType; var l = new PlanRunCollectorListener(); TestPlan plan = new TestPlan(); var verdict = new VerdictStep { VerdictOutput = verdictOutput }; BreakConditionProperty.SetBreakCondition(verdict, BreakCondition.Inherit); var verdict2 = new VerdictStep { VerdictOutput = Verdict.Pass }; plan.Steps.Add(verdict); plan.Steps.Add(verdict2); var run = plan.Execute(new[] {l}); Assert.AreEqual(finalVerdict, run.Verdict); Assert.AreEqual(runCount, l.StepRuns.Count); Assert.AreEqual(BreakCondition.Inherit, BreakConditionProperty.GetBreakCondition(verdict2)); } finally { EngineSettings.Current.AbortTestPlan = prev; } } [Test] public void EngineInheritedConditions2() { Verdict verdictOutput = Verdict.Fail; EngineSettings.AbortTestPlanType abortTestPlanType = EngineSettings.AbortTestPlanType.Step_Fail; int runCount = 1; Verdict finalVerdict = verdictOutput; var prev = EngineSettings.Current.AbortTestPlan; try { EngineSettings.Current.AbortTestPlan = abortTestPlanType; var l = new PlanRunCollectorListener(); TestPlan plan = new TestPlan(); var verdict = new VerdictStep { VerdictOutput = verdictOutput, }; BreakConditionProperty.SetBreakCondition(verdict, BreakCondition.Inherit | BreakCondition.BreakOnError); var verdict2 = new VerdictStep { VerdictOutput = Verdict.Pass }; plan.Steps.Add(verdict); plan.Steps.Add(verdict2); var run = plan.Execute(new[] {l}); Assert.AreEqual(finalVerdict, run.Verdict); Assert.AreEqual(runCount, l.StepRuns.Count); Assert.AreEqual(BreakCondition.Inherit, BreakConditionProperty.GetBreakCondition(verdict2)); } finally { EngineSettings.Current.AbortTestPlan = prev; } } /// This step overrides the verdict of the child steps. [AllowAnyChild] class VerdictOverrideStep : TestStep { public Verdict OutputVerdict { get; set; } = Verdict.Pass; public override void Run() { foreach (var step in EnabledChildSteps) RunChildStep(step, throwOnBreak: false); this.Verdict = Verdict.Pass; } } [Test] public void StepsCanOverrideVerdicts() { var plan = new TestPlan(); var stepCatch = new VerdictOverrideStep { OutputVerdict = Verdict.Pass }; var verdict = new VerdictStep { VerdictOutput = Verdict.Error }; plan.ChildTestSteps.Add(stepCatch); stepCatch.ChildTestSteps.Add(verdict); var run = plan.Execute(); Assert.AreEqual(Verdict.Pass, run.Verdict); } public class TestStepWaitInfinite : TestStep { public Semaphore Sem = new Semaphore(0,1); public override void Run() { Sem.Release(); TapThread.Sleep(TimeSpan.MaxValue); } } public class TestStepAbortPlan : TestStep { public BreakConditionsTest.TestStepWaitInfinite WaitFor { get; set; } public override void Run() { WaitFor?.Sem.WaitOne(); PlanRun.MainThread.Abort(); } } [Test] public void TestStepWaitInfiniteAndAbortTest() { var plan = new TestPlan(); var parallel = new ParallelStep(); var abort = new TestStepAbortPlan(){WaitFor = new TestStepWaitInfinite()}; parallel.ChildTestSteps.Add(abort); parallel.ChildTestSteps.Add(abort.WaitFor); plan.ChildTestSteps.Add(parallel); var run = plan.Execute(); Assert.AreEqual(Verdict.Aborted, run.Verdict); foreach (var step in parallel.ChildTestSteps) { Assert.AreEqual(Verdict.Aborted, step.Verdict); } } [Test] public void TestPlanBreakConditions() { var plan = new TestPlan(); var errorStep = new VerdictStep() {VerdictOutput = Verdict.Error}; var failStep = new VerdictStep() {VerdictOutput = Verdict.Fail}; var inconclusiveStep = new VerdictStep() {VerdictOutput = Verdict.Inconclusive}; var passStep = new VerdictStep() {VerdictOutput = Verdict.Pass}; var nopStep = new SequenceStep(); plan.Steps.Add(passStep); plan.Steps.Add(inconclusiveStep); plan.Steps.Add(failStep); plan.Steps.Add(errorStep); plan.Steps.Add(nopStep); var defaultValue = BreakConditionProperty.GetBreakCondition(plan); Assert.AreEqual(BreakCondition.Inherit, defaultValue); // break on fail, this means that 'passStep' will not get executed BreakConditionProperty.SetBreakCondition(plan, BreakCondition.BreakOnPass); var col = new PlanRunCollectorListener(); Assert.AreEqual(Verdict.Pass, plan.Execute(new []{col}).Verdict); Assert.AreEqual(1, col.StepRuns.Count); BreakConditionProperty.SetBreakCondition(plan, BreakCondition.BreakOnInconclusive); col = new PlanRunCollectorListener(); Assert.AreEqual(Verdict.Inconclusive, plan.Execute(new []{col}).Verdict); Assert.AreEqual(2, col.StepRuns.Count); BreakConditionProperty.SetBreakCondition(plan, BreakCondition.BreakOnFail); col = new PlanRunCollectorListener(); Assert.AreEqual(Verdict.Fail, plan.Execute(new []{col}).Verdict); Assert.AreEqual(3, col.StepRuns.Count); BreakConditionProperty.SetBreakCondition(plan, BreakCondition.BreakOnError); col = new PlanRunCollectorListener(); Assert.AreEqual(Verdict.Error, plan.Execute(new []{col}).Verdict); Assert.AreEqual(4, col.StepRuns.Count); BreakConditionProperty.SetBreakCondition(plan, 0); col = new PlanRunCollectorListener(); Assert.AreEqual(Verdict.Error, plan.Execute(new []{col}).Verdict); Assert.AreEqual(5, col.StepRuns.Count); } /// /// Testing that TestPlan.Locked causes BreakConditions to be locked. /// [Test] public void TestBreakConditionsLocked() { var plan = new TestPlan(); var a = AnnotationCollection.Annotate(plan); var mem = a.GetMember("BreakConditions"); Assert.IsFalse(mem.Get().IsReadOnly); plan.Locked = true; a.Read(); Assert.IsTrue(mem.Get().IsReadOnly); plan.Locked = false; a.Read(); Assert.IsFalse(mem.Get().IsReadOnly); } class TestStepThatThrows : TestStep { public override void Run() { throw new Exception("!"); } } [AllowAnyChild] class TestStepThatCatches : TestStep { public bool AvoidCatch { get; set; } public TestStepRun[] Runs; public override void Run() { if (AvoidCatch) { Runs = RunChildSteps(throwOnBreak: false).ToArray(); var run = Runs.Last(); if(run.Verdict == Verdict.Error) Assert.AreEqual("!", run.Exception.Message); else Assert.IsNull(run.Exception); Verdict = Verdict.Pass; } else { try { RunChildSteps(throwOnBreak: true); Assert.Fail("The child steps should have thrown an exception"); } catch (TestStepBreakException) { Verdict = Verdict.Pass; } catch (Exception e) { Assert.AreEqual(e.Message, "!"); Verdict = Verdict.Pass; } } } } [Test] public void TestStepThrowsException() { var a = new TestStepThatCatches(); var b = new TestStepThatThrows(); // add a 'nop' to verify a.ChildTestSteps.Add(new SequenceStep()); a.ChildTestSteps.Add(b); a.ChildTestSteps.Add(new SequenceStep()); var plan = new TestPlan(); plan.Steps.Add(a); var r = plan.Execute(); Assert.AreEqual(Verdict.Pass, r.Verdict); a.AvoidCatch = true; var r2 = plan.Execute(); Assert.AreEqual(Verdict.Pass, r2.Verdict); Assert.AreEqual(2, a.Runs.Length); } [Test] public void TestStepThrowsBreakException() { var a = new TestStepThatCatches(); var b = new VerdictStep { VerdictOutput = Verdict.Fail }; BreakConditionProperty.SetBreakCondition(b, BreakCondition.BreakOnFail); a.ChildTestSteps.Add(new SequenceStep()); a.ChildTestSteps.Add(b); a.ChildTestSteps.Add(new SequenceStep()); var plan = new TestPlan(); plan.Steps.Add(a); var r = plan.Execute(); Assert.AreEqual(Verdict.Pass, r.Verdict); a.AvoidCatch = true; var r2 = plan.Execute(); Assert.AreEqual(Verdict.Pass , r2.Verdict); Assert.AreEqual(2, a.Runs.Length); } } }