// Copyright Keysight Technologies 2012-2019 // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, you can obtain one at http://mozilla.org/MPL/2.0/. using NUnit.Framework; using System; using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Threading; using OpenTap.EngineUnitTestUtils; namespace OpenTap.Engine.UnitTests { [TestFixture] public class TestPlanCompositeRunTests { private string[] filterLog(string allLog, bool removeSpaces = false) { allLog = Regex.Replace(allLog, "[0-9]+", "0"); allLog = Regex.Replace(allLog, "0 [mnuμ]s", "0 s"); allLog = Regex.Replace(allLog, "0.0 s", "0 s"); allLog = Regex.Replace(allLog, "--+", ""); // removes -- decorators string[] allLines = allLog.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); // lines from "System" module needs to be ignores as they apear randomly in the log (happening on a background thread) allLines = allLines.Where(line => !line.Contains(": System")).ToArray(); // also remove lines from "PluginManager" module (licensing messages happening async) allLines = allLines.Where(line => !line.Contains(": PluginManager")).ToArray(); allLines = allLines.Where(line => !line.Contains(": License")).ToArray(); allLines = allLines.Where(line => !line.Contains(": Session")).ToArray(); allLines = allLines.Where(line => !line.Contains(" loaded from ")).ToArray(); allLines = allLines.Where(line => !line.Contains("No settings file exists for ")).ToArray(); if (removeSpaces) { allLines = allLines.Select(line => line.Replace(" ", "")).ToArray(); } return allLines; } [Test] public void RunCompositeLogComparison() { // ## Fragile Test // If the settings directory has been loaded in the meantime // Log messages that new files are being created will appear on first run // ComponentSettings are lazily loaded so that first happens when the test plan runs. // We do one run to clear the state of the engine. { TestPlan target2 = getTestTestPlan(); Log.Flush(); target2.PrintTestPlanRunSummary = true; target2.Execute(); Log.Flush(); } TestTraceListener trace1 = new TestTraceListener(); Log.AddListener(trace1); TestPlan target = getTestTestPlan(); target.PrintTestPlanRunSummary = true; target.Execute(); Log.RemoveListener(trace1); TestTraceListener trace2 = new TestTraceListener(); Log.AddListener(trace2); target = getTestTestPlan(); target.PrintTestPlanRunSummary = true; target.Open(); target.Execute(); target.Close(); Log.RemoveListener(trace2); string allLog1 = trace1.allLog.ToString(); string allLog2 = trace2.allLog.ToString(); string[] log1Lines = filterLog(allLog1); string[] log2Lines = filterLog(allLog2); string[] log2LinesNoSpaces = filterLog(allLog2, true); Assert.AreEqual(log1Lines.Count() + 2, log2Lines.Count(), allLog1 + Environment.NewLine + "##########" + Environment.NewLine + allLog2); for (int i = 0; i < log1Lines.Length; i++) { var line = log1Lines[i].Replace(" ", ""); if (!log2LinesNoSpaces.Contains(line)) // We compare lines with removed spaces to avoid flakyness in CI. { // Print actual comparison data Console.WriteLine($"Could not find '{line}' in following logs:"); foreach (var linez in log2LinesNoSpaces) Console.WriteLine($"{linez}"); Console.WriteLine($"--------------- Printing logs without spaces removed ---------------"); // Print log data without their spaces removed Console.WriteLine($"First run logs:"); foreach (var linez in log1Lines) Console.WriteLine($"- {linez}"); Console.WriteLine($"Second run logs:"); foreach (var linez in log2Lines) Console.WriteLine($"- {linez}"); Assert.Fail($"The logs from two testplan executions does not match..."); } } } [Test] public void ShortTimeSpanTest() { var v = new[] { 0.01, 0.001, 0.0001, 10e-6, 1e-6, 1e-9, 1e-10 }; var s = new[] { "10 ms", "1 ms", "100 us", "10 μs", "1 μs", "1 ns", "0 s" }; for (int i = 0; i < v.Length; i++) { var s1 = ShortTimeSpan.FromSeconds(v[i]); var s2 = ShortTimeSpan.FromString(s[i]); StringAssert.AreEqualIgnoringCase(s1.ToString(), s2.ToString()); } } [Test] public void LongTimeSpanTest() { var seconds = new TimeSpan[] { new TimeSpan(1, 0, 0, 0), new TimeSpan(0, 23, 30, 0) , new TimeSpan(23, 12, 22, 0), new TimeSpan(1, 0, 5, 0), new TimeSpan(0, 5, 0, 3), new TimeSpan(3, 0, 21, 10), new TimeSpan(999, 23 ,59, 59), new TimeSpan(1, 0,0, 59), new TimeSpan(0, 0,0, 1), new TimeSpan(0,0, 0,0), new TimeSpan(0,0, 1,59), new TimeSpan(0,0,0,59) }; var expectedTimeSpan = new[] { "1 d", "23 h 30 m", "23 d 12 h 22 m", "1 d 5 m", "5 h 3 s" , "3 d 21 m 10 s", "999 d 23 h 59 m 59 s", "1 d 59 s", "1.00 s", "0.00 s", "1 m 59 s", "59.0 s" }; for (int i = 0; i < seconds.Length; i++) { var actualTimeSpan = ShortTimeSpan.LongTimeSpanFormat(seconds[i]).Trim(); StringAssert.AreEqualIgnoringCase(expectedTimeSpan[i], actualTimeSpan); } } [Test, Retry(3)] public void RunCompositeFollowedByRun() { TestPlan target = getTestTestPlan(); target.Open(); target.Execute(); target.Close(); Log.Flush(); TestTraceListener trace2 = new TestTraceListener(); Log.AddListener(trace2); target.Open(); target.Execute(); target.Close(); Log.RemoveListener(trace2); TestTraceListener trace1 = new TestTraceListener(); Log.AddListener(trace1); target.Execute(); Log.RemoveListener(trace1); string allLog1 = trace1.allLog.ToString(); string allLog2 = trace2.allLog.ToString(); string[] log1Lines = filterLog(allLog1); string[] log2Lines = filterLog(allLog2); Assert.AreEqual(log1Lines.Count() + 2, log2Lines.Count(), allLog2); for (int i = 0; i < log1Lines.Length; i++) { CollectionAssert.Contains(log2Lines, log1Lines[i]); } } [Test] public void RunCompositeOpenClose() { TestPlan target = getTestTestPlan(); target.Open(); target.Close(); target.Open(); target.Close(); } [Test] public void RunCompositeRunTwice() { TestTraceListener trace = new TestTraceListener(); Log.AddListener(trace); TestPlan target = getTestTestPlan(); target.Open(); target.Execute(); target.Execute(); target.Close(); Log.RemoveListener(trace); trace.AssertErrors(); TestTestStep step = target.Steps[0] as TestTestStep; Assert.AreEqual(2, step.PrePlanRunCount, "PrePlanRun was not called the correct number of times."); Assert.AreEqual(2, step.PostPlanRunCount, "PostPlanRun was not called the correct number of times."); } [Test] public void RunCompositeCloseWithoutOpen() { TestTraceListener trace = new TestTraceListener(); Log.AddListener(trace); TestPlan target = getTestTestPlan(); Assert.Throws(typeof(InvalidOperationException), target.Close); Log.RemoveListener(trace); } [Test] public void RunCompositeCloseWhileRunning() { TestTraceListener trace = new TestTraceListener(); Log.AddListener(trace); TestPlan target = getTestTestPlan(); var t1 = Task.Factory.StartNew(() => target.Execute()); try { Thread.Sleep(30); Assert.Throws(typeof(InvalidOperationException),target.Close); } finally { Log.RemoveListener(trace); t1.Wait(); } } [Test] public void RunCompositeStartTime() { PlanRunCollectorListener listener = new PlanRunCollectorListener(); ResultSettings.Current.Add(listener); TestTraceListener trace = new TestTraceListener(); Log.AddListener(trace); TestPlan target = getTestTestPlan(); target.Open(); target.Execute(); listener.StepRuns.Clear(); target.Execute(); target.Close(); Log.RemoveListener(trace); ResultSettings.Current.Remove(listener); Assert.AreEqual(2, listener.PlanRuns.Select(run => run.StartTimeStamp).Distinct().Count()); Assert.AreEqual(1, listener.StepRuns.Count()); } [Test] public void RunCompositeMetaData() { PlanRunCollectorListener listener = new PlanRunCollectorListener(); ResultSettings.Current.Add(listener); TestTraceListener trace = new TestTraceListener(); Log.AddListener(trace); TestPlan target = getTestTestPlan(); TestPlanRun run = target.Execute(); Log.RemoveListener(trace); ResultSettings.Current.Remove(listener); Assert.IsTrue(run.Parameters.Any(par => par.Value.ToString() == "Test Instrument")); Assert.IsFalse(run.Parameters.Any(par => par.Name == "Comment")); } [Test] public void RunCompositeMetaData2() { PlanRunCollectorListener listener = new PlanRunCollectorListener(); ResultSettings.Current.Add(listener); TestTraceListener trace = new TestTraceListener(); Log.AddListener(trace); TestPlan target = getTestTestPlan(); target.Open(); target.Execute(); target.Execute(); target.Close(); Log.RemoveListener(trace); ResultSettings.Current.Remove(listener); Assert.IsTrue(listener.PlanRuns.First().Parameters.Any(par => par.Value.ToString() == "Test Instrument")); Assert.AreEqual(1, listener.PlanRuns.Last().Parameters.Count(par => par.Value.ToString() == "Test Instrument")); } [Test] public void RunCompositeAddInstrumentAfterOpen() { PlanRunCollectorListener listener = new PlanRunCollectorListener(); ResultSettings.Current.Add(listener); TestTraceListener trace = new TestTraceListener(); Log.AddListener(trace); TestPlan target = getTestTestPlan(); target.Open(); TestInstrument instr = InstrumentSettings.Current.FirstOrDefault(i => i is TestInstrument && (i as TestInstrument).Name.EndsWith("2")) as TestInstrument; if (instr == null) { instr = new TestInstrument { Name = "Test Instrument 2" }; InstrumentSettings.Current.Add(instr); } (target.Steps[0] as TestTestStep).Instr = instr; target.Execute(); listener.StepRuns.Clear(); target.Execute(); target.Close(); Log.RemoveListener(trace); ResultSettings.Current.Remove(listener); Assert.AreEqual(2, listener.PlanRuns.Select(run => run.StartTimeStamp).Distinct().Count()); Assert.AreEqual(1, listener.StepRuns.Count()); } #region Helper Methods and Classes/Plugins private TestPlan getTestTestPlan() { if (!InstrumentSettings.Current.Any(instr => instr is TestInstrument)) InstrumentSettings.Current.Add(new TestInstrument()); TestPlan target = new TestPlan(); target.Steps.Add(new TestPlanCompositeRunTests.TestTestStep()); return target; } public class TestTestStep : TestStep { public TestInstrument Instr { get; set; } public int PostPlanRunCount = 0; public int PrePlanRunCount = 0; public override void PrePlanRun() { PrePlanRunCount++; } public override void Run() { if (!Instr.isOpen) throw new Exception("TestStep run before instrument was opened."); Log.Info("Running Test TestStep."); } public override void PostPlanRun() { PostPlanRunCount++; } } public class TestInstrument : OpenTap.Instrument { [MetaData] public new string Name { get; set; } //[MetaData(MetaDataOptions.Ignore)] public string Comment { get; set; } public TestInstrument() { Name = "Test Instrument"; } public bool isOpen = false; public override void Open() { base.Open(); TapThread.Sleep(50); isOpen = true; Log.Info(Name + " Opened."); } public override void Close() { if (!isOpen) throw new Exception("Instrument was closed but not opened."); isOpen = false; base.Close(); Log.Info(Name + " Closed."); } } #endregion } }