// Copyright Keysight Technologies 2012-2019
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, you can obtain one at http://mozilla.org/MPL/2.0/.
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading;
namespace OpenTap.Cli
{
internal enum ExitStatus : int
{
TestPlanInconclusive = 20,
TestPlanFail = 30,
TestPlanError = 50,
LoadError = 70,
}
///
/// Test plan run CLI action. Execute a test plan with 'tap.exe run test.TapPlan'
///
[Display("run", Description: "Run a test plan.")]
public class RunCliAction : ICliAction
{
///
/// Specify a bench settings profile from which to load the bench settings. The parameter given here should correspond to the name of a subdirectory of %TAP_PATH%/Settings/Bench. If not specified, %TAP_PATH%/Settings/Bench/Default is used.
///
[CommandLineArgument("settings", Description = "Specify a bench settings profile from which to load\nthe bench settings. The parameter given here should correspond\nto the name of a subdirectory of /Settings/Bench.\nIf not specified, /Settings/Bench/Default is used.")]
public string Settings { get; set; } = "";
///
/// Additional directories to be searched for plugins. This option may be used multiple times, e.g., --search dir1 --search dir2.
///
[CommandLineArgument("search", Description = "Additional directories to be searched for plugins.\nThis option may be used multiple times, e.g., --search dir1 --search dir2.")]
[Browsable(false)]
public string[] Search { get; set; } = new string[0];
///
/// Set a resource metadata parameter. Use the syntax parameter=value, e.g., --metadata dut-id=5. This option may be used multiple times.
///
[CommandLineArgument("metadata", Description = "Set a resource metadata parameter.\nUse the syntax parameter=value, e.g., --metadata dut-id=5.\nThis option may be used multiple times.")]
public string[] Metadata { get; set; } = new string[0];
///
/// Never prompt for user input.
///
[CommandLineArgument("non-interactive", Description = "Never prompt for user input.")]
public bool NonInteractive { get; set; } = false;
///
/// Set an external test plan parameter. Use the syntax parameter=value, e.g., -e delay=1.0. This option may be used multiple times, or a .csv file containing a \"parameter, value\" pair on each line can be specified as -e file.csv.
///
[CommandLineArgument("external", ShortName = "e", Description = "Set an external test plan parameter.\nUse the syntax parameter=value, e.g., -e delay=1.0.\nThis option may be used multiple times, or a .csv file containing a\n\"parameter, value\" pair on each line can be specified as -e file.csv.")]
public string[] External { get; set; } = new string[0];
///
/// Try setting an external test plan parameter, ignoring errors if it does not exist in the test plan. Use the syntax parameter=value, e.g., -t delay=1.0. This option may be used multiple times
///
[CommandLineArgument("try-external", ShortName = "t", Description = "Try setting an external test plan parameter,\nignoring errors if it does not exist in the test plan.\nUse the syntax parameter=value, e.g., -t delay=1.0.\nThis option may be used multiple times.")]
public string[] TryExternal { get; set; } = new string[0];
///
/// List the available external test plan parameters.
///
[CommandLineArgument("list-external-parameters", Description = "List the available external test plan parameters.")]
public bool ListExternal { get; set; } = false;
///
/// Enable a subset of the currently configured result listeners given as a comma-separated list, e.g., --results SQLite,CSV. To disable all result listeners use --results \"\".
///
[CommandLineArgument("results", Description = "Enable a subset of the currently configured result listeners\ngiven as a comma-separated list, e.g., --results SQLite,CSV.\nTo disable all result listeners use --results \"\".")]
public string Results { get; set; }
///
/// Ignore the errors for deserialization of test plan
///
[CommandLineArgument("ignore-load-errors", Description = "Ignore the errors during loading of test plan.")]
public bool IgnoreLoadErrors { get; set; } = false;
///
/// Location of test plan to be executed.
///
[UnnamedCommandLineArgument("Test Plan", Required = true, Description = "The test plan to run.")]
public string TestPlanPath { get; set; } = "";
///
/// Log to write debug/trace messages to
///
private static readonly TraceSource log = Log.CreateSource("Main");
private static TestPlan Plan;
///
/// Executes test plan
///
///
public int Execute(CancellationToken cancellationToken)
{
List metaData = new List();
HandleMetadata(metaData);
string planToLoad = null;
// If the --search argument is used, add the --ignore-load-errors to fix any load issues.
if (Search.Any())
{
// Warn
log.Warning("Argument '--search' is deprecated. The '--ignore-load-errors' argument has been added to avoid potential test plan load issues.");
IgnoreLoadErrors = true;
}
try
{
planToLoad = !string.IsNullOrWhiteSpace(TestPlanPath) ? Path.GetFullPath(TestPlanPath) : null;
}
catch (Exception)
{
Console.WriteLine("Invalid path: '{0}'", TestPlanPath);
Console.WriteLine("The path only supports a valid file path.");
log.Info("Unable to continue. Now exiting tap.exe.");
return (int)ExitCodes.ArgumentError;
}
try
{
HandleSearchDirectories();
}
catch (ArgumentException)
{
log.Info("Unable to continue. Now exiting tap.exe.");
return (int)ExitCodes.ArgumentError;
}
EngineSettings.LoadWorkingDirectory(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
Assembly assembly = Assembly.GetExecutingAssembly();
Console.WriteLine($"OpenTAP Command Line Interface {FileVersionInfo.GetVersionInfo(assembly.Location).ProductVersion}\n");
if (!string.IsNullOrWhiteSpace(Settings))
{
TestPlanRunner.SetSettingsDir(Settings);
}
if (!NonInteractive && UserInput.Interface == null)
{
CliUserInputInterface.Load();
}
if (Results != null)
{
try
{
HandleResultListeners();
}
catch (ArgumentException)
{
log.Info("Unable to continue. Now exiting tap.exe.");
return (int)ExitCodes.ArgumentError;
}
}
if (planToLoad == null)
{
Console.WriteLine("Please supply a valid test plan path as an argument.");
log.Info("Unable to continue. Now exiting tap.exe.");
return (int)ExitCodes.ArgumentError;
}
// Load TestPlan:
if (!File.Exists(planToLoad))
{
log.Error("File '{0}' does not exist.", planToLoad);
log.Flush();
Thread.Sleep(100);
log.Info("Unable to continue. Now exiting tap.exe.");
return (int)ExitCodes.ArgumentError;
}
try
{
HandleExternalParametersAndLoadPlan(planToLoad);
}
catch (TestPlan.PlanLoadException ex)
{
// at this point the log messages are already written out saying what went wrong.
log.Error("Unable to load test plan.");
log.Debug(ex);
return (int)ExitStatus.LoadError;
}
catch (OperationCanceledException) when (TapThread.Current.AbortToken.IsCancellationRequested)
{
log.Info("tap.exe was exited due to an interrupt. (CTRL+C)");
return (int)ExitCodes.UserCancelled;
}
catch (ArgumentException ex)
{
if(!string.IsNullOrWhiteSpace(ex.Message))
log.Error(ex.Message);
return (int)ExitCodes.ArgumentError;
}
catch (Exception e)
{
log.Error("Caught error while loading test plan: '{0}'", e.Message);
log.Debug(e);
return (int)ExitStatus.LoadError;
}
log.Info("Test Plan: {0}", Plan.Name);
if (ListExternal)
{
PrintExternalParameters(log);
return (int)ExitCodes.Success;
}
Verdict verdict = TestPlanRunner.RunPlanForDut(Plan, metaData, cancellationToken);
if (TapThread.Current.AbortToken.IsCancellationRequested)
{
log.Info("tap.exe was exited due to an interrupt. (CTRL+C)");
return (int)ExitCodes.UserCancelled;
}
if (verdict == Verdict.Inconclusive)
return (int)ExitStatus.TestPlanInconclusive;
if (verdict == Verdict.Fail)
return (int)ExitStatus.TestPlanFail;
if (verdict > Verdict.Fail)
return (int)ExitStatus.TestPlanError;
return (int)ExitCodes.Success;
}
private void HandleExternalParametersAndLoadPlan(string planToLoad)
{
List values = new List();
var serializer = new TapSerializer();
var extparams = serializer.GetSerializer();
if (External.Length > 0)
values.AddRange(External);
if (TryExternal.Length > 0)
values.AddRange(TryExternal);
Plan = new TestPlan();
List externalParameterFiles = new List();
foreach (var externalParam in values)
{
int equalIdx = externalParam.IndexOf('=');
if (equalIdx == -1)
{
externalParameterFiles.Add(externalParam);
continue;
}
var name = externalParam.Substring(0, equalIdx);
var value = externalParam.Substring(equalIdx + 1);
extparams.PreloadedValues[name] = value;
}
var log = Log.CreateSource("CLI");
var timer = Stopwatch.StartNew();
using (var fs = new FileStream(planToLoad, FileMode.Open, FileAccess.Read))
{
// only cache the XML if there are no external parameters.
bool cacheXml = values.Any() == false && externalParameterFiles.Any() == false;
Plan = TestPlan.Load(fs, planToLoad, cacheXml, serializer, IgnoreLoadErrors);
log.Info(timer, "Loaded test plan from {0}", planToLoad);
}
if (externalParameterFiles.Count > 0)
{
var importers = CreateInstances();
var CurDir = Directory.GetCurrentDirectory();
try
{
Directory.SetCurrentDirectory(EngineSettings.StartupDir);
foreach (var file in externalParameterFiles)
{
var ext = Path.GetExtension(file);
log.Info($"Loading external parameters from '{file}'.");
var importer = importers
// find importer that accepts that extension, case-insensitively.
.FirstOrDefault(i => string.Equals(i.Extension, ext, StringComparison.OrdinalIgnoreCase));
if (importer != null)
{
importer.ImportExternalParameters(Plan, file);
}
else
{
throw new ArgumentException($"No installed plugins provide loading of external parameters from '{ext}' files. No external parameters loaded from '{file}'.");
}
}
}
finally
{
Directory.SetCurrentDirectory(CurDir);
}
}
if (External.Length > 0)
{ // Print warnings if an --external parameter was not in the test plan.
foreach (var externalParam in External)
{
var equalIdx = externalParam.IndexOf('=');
if (equalIdx == -1) continue;
var name = externalParam.Substring(0, equalIdx);
if (Plan.ExternalParameters.Get(name) != null) continue;
log.Warning("External parameter '{0}' does not exist in the test plan.", name);
log.Warning("Statement '{0}' has no effect.", externalParam);
throw new ArgumentException("");
}
}
}
private void PrintExternalParameters(TraceSource log)
{
var annotation = AnnotationCollection.Annotate(Plan).Get();
log.Info("Listing {0} External Test Plan Parameters:", Plan.ExternalParameters.Entries.Count);
foreach (var member in annotation.Members)
{
if (member.Get()?.Member is ParameterMemberData param)
{
var multiValues = member.Get()?.SelectedValues;
string printStr = "";
if (multiValues != null)
{
foreach (var val in multiValues)
printStr += string.Format("{0} | ", val.Get()?.Value ?? val.Get()?.Value?.ToString() ?? "");
printStr = printStr.Remove(printStr.Length - 3); // Remove trailing delimiter
}
else
printStr = member.Get()?.Value ?? member.Get()?.Value?.ToString();
log.Info(" {0} = {1}", param.Name, printStr);
if (member.Get() is IAvailableValuesAnnotationProxy avail)
{
log.Info(" Available Values:");
foreach (var val in avail.AvailableValues ?? new AnnotationCollection[0])
log.Info(" {0}", val.Get()?.Value ?? val.Get()?.Value?.ToString() ?? "");
}
}
}
}
private static List CreateInstances()
{
var externalParameterExportPlugins = PluginManager.GetPlugins();
var fileHandlers = new List();
foreach (var plugin in externalParameterExportPlugins)
{
fileHandlers.Add((T)Activator.CreateInstance(plugin));
}
return fileHandlers;
}
private void HandleResultListeners()
{
foreach (var r in ResultSettings.Current.OfType())
r.IsEnabled = false;
var rs = Results.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries).ToList();
foreach (var name in rs.ToList())
{
bool foundOne = false;
foreach (var r in ResultSettings.Current.OfType())
{
if (string.Compare(r.Name, name, true) == 0)
{
r.IsEnabled = true;
foundOne = true;
}
}
if (foundOne)
rs.Remove(name);
}
if (rs.Count > 0)
{
Console.Error.WriteLine("Unknown result listeners: {0}", string.Join(",", rs));
Console.WriteLine("Known result listeners are:");
}
foreach (var r in ResultSettings.Current.OfType().ToList())
Console.WriteLine("[{2}] {0}: {1}", r.Name, r.ToString(), r.IsEnabled ? "x" : " ");
if (rs.Count > 0)
throw new ArgumentException();
}
private void HandleSearchDirectories()
{
if (Search.Length > 0)
{
for (int i = 0; i < Search.Length; i++)
{
string dir = Search[i];
try
{
string fullDir = Path.GetFullPath(dir);
if (!Directory.Exists(fullDir))
{
Console.WriteLine("Invalid plugin search path: '{0}'", fullDir);
throw new ArgumentException();
}
Search[i] = fullDir;
}
catch (ArgumentException)
{
Console.WriteLine("Invalid plugin search path: '{0}'", dir);
throw;
}
}
PluginManager.DirectoriesToSearch.AddRange(Search);
PluginManager.SearchAsync();
}
}
private void HandleMetadata(List metaData)
{
if (Metadata.Length > 0)
{
foreach (string data in Metadata)
{
string[] eql = data.Split('=');
if (eql.Length != 2)
{
log.Warning("Unable to parse metadata parameter '{0}'", data);
continue;
}
metaData.Add(new ResultParameter("", eql[0], eql[1], metadata: new MetaDataAttribute(false, eql[0])));
}
}
}
}
}