// Copyright Keysight Technologies 2012-2019 // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, you can obtain one at http://mozilla.org/MPL/2.0/. using System; using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; using System.IO; using System.Linq; using System.Reflection; using System.Threading; namespace OpenTap.Cli { internal enum ExitStatus : int { TestPlanInconclusive = 20, TestPlanFail = 30, TestPlanError = 50, LoadError = 70, } /// /// Test plan run CLI action. Execute a test plan with 'tap.exe run test.TapPlan' /// [Display("run", Description: "Run a test plan.")] public class RunCliAction : ICliAction { /// /// Specify a bench settings profile from which to load the bench settings. The parameter given here should correspond to the name of a subdirectory of %TAP_PATH%/Settings/Bench. If not specified, %TAP_PATH%/Settings/Bench/Default is used. /// [CommandLineArgument("settings", Description = "Specify a bench settings profile from which to load\nthe bench settings. The parameter given here should correspond\nto the name of a subdirectory of /Settings/Bench.\nIf not specified, /Settings/Bench/Default is used.")] public string Settings { get; set; } = ""; /// /// Additional directories to be searched for plugins. This option may be used multiple times, e.g., --search dir1 --search dir2. /// [CommandLineArgument("search", Description = "Additional directories to be searched for plugins.\nThis option may be used multiple times, e.g., --search dir1 --search dir2.")] [Browsable(false)] public string[] Search { get; set; } = new string[0]; /// /// Set a resource metadata parameter. Use the syntax parameter=value, e.g., --metadata dut-id=5. This option may be used multiple times. /// [CommandLineArgument("metadata", Description = "Set a resource metadata parameter.\nUse the syntax parameter=value, e.g., --metadata dut-id=5.\nThis option may be used multiple times.")] public string[] Metadata { get; set; } = new string[0]; /// /// Never prompt for user input. /// [CommandLineArgument("non-interactive", Description = "Never prompt for user input.")] public bool NonInteractive { get; set; } = false; /// /// Set an external test plan parameter. Use the syntax parameter=value, e.g., -e delay=1.0. This option may be used multiple times, or a .csv file containing a \"parameter, value\" pair on each line can be specified as -e file.csv. /// [CommandLineArgument("external", ShortName = "e", Description = "Set an external test plan parameter.\nUse the syntax parameter=value, e.g., -e delay=1.0.\nThis option may be used multiple times, or a .csv file containing a\n\"parameter, value\" pair on each line can be specified as -e file.csv.")] public string[] External { get; set; } = new string[0]; /// /// Try setting an external test plan parameter, ignoring errors if it does not exist in the test plan. Use the syntax parameter=value, e.g., -t delay=1.0. This option may be used multiple times /// [CommandLineArgument("try-external", ShortName = "t", Description = "Try setting an external test plan parameter,\nignoring errors if it does not exist in the test plan.\nUse the syntax parameter=value, e.g., -t delay=1.0.\nThis option may be used multiple times.")] public string[] TryExternal { get; set; } = new string[0]; /// /// List the available external test plan parameters. /// [CommandLineArgument("list-external-parameters", Description = "List the available external test plan parameters.")] public bool ListExternal { get; set; } = false; /// /// Enable a subset of the currently configured result listeners given as a comma-separated list, e.g., --results SQLite,CSV. To disable all result listeners use --results \"\". /// [CommandLineArgument("results", Description = "Enable a subset of the currently configured result listeners\ngiven as a comma-separated list, e.g., --results SQLite,CSV.\nTo disable all result listeners use --results \"\".")] public string Results { get; set; } /// /// Ignore the errors for deserialization of test plan /// [CommandLineArgument("ignore-load-errors", Description = "Ignore the errors during loading of test plan.")] public bool IgnoreLoadErrors { get; set; } = false; /// /// Location of test plan to be executed. /// [UnnamedCommandLineArgument("Test Plan", Required = true, Description = "The test plan to run.")] public string TestPlanPath { get; set; } = ""; /// /// Log to write debug/trace messages to /// private static readonly TraceSource log = Log.CreateSource("Main"); private static TestPlan Plan; /// /// Executes test plan /// /// public int Execute(CancellationToken cancellationToken) { List metaData = new List(); HandleMetadata(metaData); string planToLoad = null; // If the --search argument is used, add the --ignore-load-errors to fix any load issues. if (Search.Any()) { // Warn log.Warning("Argument '--search' is deprecated. The '--ignore-load-errors' argument has been added to avoid potential test plan load issues."); IgnoreLoadErrors = true; } try { planToLoad = !string.IsNullOrWhiteSpace(TestPlanPath) ? Path.GetFullPath(TestPlanPath) : null; } catch (Exception) { Console.WriteLine("Invalid path: '{0}'", TestPlanPath); Console.WriteLine("The path only supports a valid file path."); log.Info("Unable to continue. Now exiting tap.exe."); return (int)ExitCodes.ArgumentError; } try { HandleSearchDirectories(); } catch (ArgumentException) { log.Info("Unable to continue. Now exiting tap.exe."); return (int)ExitCodes.ArgumentError; } EngineSettings.LoadWorkingDirectory(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)); Assembly assembly = Assembly.GetExecutingAssembly(); Console.WriteLine($"OpenTAP Command Line Interface {FileVersionInfo.GetVersionInfo(assembly.Location).ProductVersion}\n"); if (!string.IsNullOrWhiteSpace(Settings)) { TestPlanRunner.SetSettingsDir(Settings); } if (!NonInteractive && UserInput.Interface == null) { CliUserInputInterface.Load(); } if (Results != null) { try { HandleResultListeners(); } catch (ArgumentException) { log.Info("Unable to continue. Now exiting tap.exe."); return (int)ExitCodes.ArgumentError; } } if (planToLoad == null) { Console.WriteLine("Please supply a valid test plan path as an argument."); log.Info("Unable to continue. Now exiting tap.exe."); return (int)ExitCodes.ArgumentError; } // Load TestPlan: if (!File.Exists(planToLoad)) { log.Error("File '{0}' does not exist.", planToLoad); log.Flush(); Thread.Sleep(100); log.Info("Unable to continue. Now exiting tap.exe."); return (int)ExitCodes.ArgumentError; } try { HandleExternalParametersAndLoadPlan(planToLoad); } catch (TestPlan.PlanLoadException ex) { // at this point the log messages are already written out saying what went wrong. log.Error("Unable to load test plan."); log.Debug(ex); return (int)ExitStatus.LoadError; } catch (OperationCanceledException) when (TapThread.Current.AbortToken.IsCancellationRequested) { log.Info("tap.exe was exited due to an interrupt. (CTRL+C)"); return (int)ExitCodes.UserCancelled; } catch (ArgumentException ex) { if(!string.IsNullOrWhiteSpace(ex.Message)) log.Error(ex.Message); return (int)ExitCodes.ArgumentError; } catch (Exception e) { log.Error("Caught error while loading test plan: '{0}'", e.Message); log.Debug(e); return (int)ExitStatus.LoadError; } log.Info("Test Plan: {0}", Plan.Name); if (ListExternal) { PrintExternalParameters(log); return (int)ExitCodes.Success; } Verdict verdict = TestPlanRunner.RunPlanForDut(Plan, metaData, cancellationToken); if (TapThread.Current.AbortToken.IsCancellationRequested) { log.Info("tap.exe was exited due to an interrupt. (CTRL+C)"); return (int)ExitCodes.UserCancelled; } if (verdict == Verdict.Inconclusive) return (int)ExitStatus.TestPlanInconclusive; if (verdict == Verdict.Fail) return (int)ExitStatus.TestPlanFail; if (verdict > Verdict.Fail) return (int)ExitStatus.TestPlanError; return (int)ExitCodes.Success; } private void HandleExternalParametersAndLoadPlan(string planToLoad) { List values = new List(); var serializer = new TapSerializer(); var extparams = serializer.GetSerializer(); if (External.Length > 0) values.AddRange(External); if (TryExternal.Length > 0) values.AddRange(TryExternal); Plan = new TestPlan(); List externalParameterFiles = new List(); foreach (var externalParam in values) { int equalIdx = externalParam.IndexOf('='); if (equalIdx == -1) { externalParameterFiles.Add(externalParam); continue; } var name = externalParam.Substring(0, equalIdx); var value = externalParam.Substring(equalIdx + 1); extparams.PreloadedValues[name] = value; } var log = Log.CreateSource("CLI"); var timer = Stopwatch.StartNew(); using (var fs = new FileStream(planToLoad, FileMode.Open, FileAccess.Read)) { // only cache the XML if there are no external parameters. bool cacheXml = values.Any() == false && externalParameterFiles.Any() == false; Plan = TestPlan.Load(fs, planToLoad, cacheXml, serializer, IgnoreLoadErrors); log.Info(timer, "Loaded test plan from {0}", planToLoad); } if (externalParameterFiles.Count > 0) { var importers = CreateInstances(); var CurDir = Directory.GetCurrentDirectory(); try { Directory.SetCurrentDirectory(EngineSettings.StartupDir); foreach (var file in externalParameterFiles) { var ext = Path.GetExtension(file); log.Info($"Loading external parameters from '{file}'."); var importer = importers // find importer that accepts that extension, case-insensitively. .FirstOrDefault(i => string.Equals(i.Extension, ext, StringComparison.OrdinalIgnoreCase)); if (importer != null) { importer.ImportExternalParameters(Plan, file); } else { throw new ArgumentException($"No installed plugins provide loading of external parameters from '{ext}' files. No external parameters loaded from '{file}'."); } } } finally { Directory.SetCurrentDirectory(CurDir); } } if (External.Length > 0) { // Print warnings if an --external parameter was not in the test plan. foreach (var externalParam in External) { var equalIdx = externalParam.IndexOf('='); if (equalIdx == -1) continue; var name = externalParam.Substring(0, equalIdx); if (Plan.ExternalParameters.Get(name) != null) continue; log.Warning("External parameter '{0}' does not exist in the test plan.", name); log.Warning("Statement '{0}' has no effect.", externalParam); throw new ArgumentException(""); } } } private void PrintExternalParameters(TraceSource log) { var annotation = AnnotationCollection.Annotate(Plan).Get(); log.Info("Listing {0} External Test Plan Parameters:", Plan.ExternalParameters.Entries.Count); foreach (var member in annotation.Members) { if (member.Get()?.Member is ParameterMemberData param) { var multiValues = member.Get()?.SelectedValues; string printStr = ""; if (multiValues != null) { foreach (var val in multiValues) printStr += string.Format("{0} | ", val.Get()?.Value ?? val.Get()?.Value?.ToString() ?? ""); printStr = printStr.Remove(printStr.Length - 3); // Remove trailing delimiter } else printStr = member.Get()?.Value ?? member.Get()?.Value?.ToString(); log.Info(" {0} = {1}", param.Name, printStr); if (member.Get() is IAvailableValuesAnnotationProxy avail) { log.Info(" Available Values:"); foreach (var val in avail.AvailableValues ?? new AnnotationCollection[0]) log.Info(" {0}", val.Get()?.Value ?? val.Get()?.Value?.ToString() ?? ""); } } } } private static List CreateInstances() { var externalParameterExportPlugins = PluginManager.GetPlugins(); var fileHandlers = new List(); foreach (var plugin in externalParameterExportPlugins) { fileHandlers.Add((T)Activator.CreateInstance(plugin)); } return fileHandlers; } private void HandleResultListeners() { foreach (var r in ResultSettings.Current.OfType()) r.IsEnabled = false; var rs = Results.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries).ToList(); foreach (var name in rs.ToList()) { bool foundOne = false; foreach (var r in ResultSettings.Current.OfType()) { if (string.Compare(r.Name, name, true) == 0) { r.IsEnabled = true; foundOne = true; } } if (foundOne) rs.Remove(name); } if (rs.Count > 0) { Console.Error.WriteLine("Unknown result listeners: {0}", string.Join(",", rs)); Console.WriteLine("Known result listeners are:"); } foreach (var r in ResultSettings.Current.OfType().ToList()) Console.WriteLine("[{2}] {0}: {1}", r.Name, r.ToString(), r.IsEnabled ? "x" : " "); if (rs.Count > 0) throw new ArgumentException(); } private void HandleSearchDirectories() { if (Search.Length > 0) { for (int i = 0; i < Search.Length; i++) { string dir = Search[i]; try { string fullDir = Path.GetFullPath(dir); if (!Directory.Exists(fullDir)) { Console.WriteLine("Invalid plugin search path: '{0}'", fullDir); throw new ArgumentException(); } Search[i] = fullDir; } catch (ArgumentException) { Console.WriteLine("Invalid plugin search path: '{0}'", dir); throw; } } PluginManager.DirectoriesToSearch.AddRange(Search); PluginManager.SearchAsync(); } } private void HandleMetadata(List metaData) { if (Metadata.Length > 0) { foreach (string data in Metadata) { string[] eql = data.Split('='); if (eql.Length != 2) { log.Warning("Unable to parse metadata parameter '{0}'", data); continue; } metaData.Add(new ResultParameter("", eql[0], eql[1], metadata: new MetaDataAttribute(false, eql[0]))); } } } } }