// Copyright Keysight Technologies 2012-2019 // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, you can obtain one at http://mozilla.org/MPL/2.0/. using System; using System.Collections.Generic; using System.ComponentModel; using System.IO; using System.Linq; using System.Reflection; using System.Diagnostics; using System.IO.Compression; using Tap.Shared; using System.Xml.Serialization; using OpenTap.Cli; namespace OpenTap.Package { internal static class PackageDefExt { // // Note: There is some code duplication between PackageDefExt and PackageDef, // but usually PackageDefExt does something in addition to what PackageDef does. // static TraceSource log = Log.CreateSource("Package"); private static IEnumerable GetSupportedModels(ITypeData td) { var attrs = td.GetAttributes().GroupBy(x => x.Manufacturer); foreach (var grp in attrs) { var models = grp.SelectMany(x => x.Models); yield return new SupportedModelsAttribute(grp.Key, models.ToArray()); } } private static void EnumeratePlugins(PackageDef pkg, List searchedAssemblies) { foreach (PackageFile def in pkg.Files) { if (def.Plugins == null || !def.Plugins.Any()) { if (File.Exists(def.FileName) && IsDotNetAssembly(def.FileName)) { try { // if the file is already in its destination dir use that instead. // That file is much more likely to be inside the OpenTAP dir we already searched. string fullPath = Path.GetFullPath(def.FileName); // Find the file in searchedAssemblies using its name+version because // searchedAssemblies will only contain AssemblyInfos with Distinct FullNames AssemblyName name = AssemblyName.GetAssemblyName(fullPath); AssemblyData assembly = searchedAssemblies.FirstOrDefault(a => a.Name == name.Name && a.Version == name.Version); if (assembly != null) { var otherversions = searchedAssemblies.Where(a => a.Name == assembly.Name).ToList(); if (otherversions.Count > 1) { // if SourcePath is set to a location inside the installation folder, the file will // be there twice at this point. This is expected. bool isOtherVersionFromCopy = otherversions.Count == 2 && otherversions.Any(a => a.Location == Path.GetFullPath(def.FileName)) && otherversions.Any(a => a.Location == Path.GetFullPath(def.RelativeDestinationPath)); if (!isOtherVersionFromCopy) log.Warning($"Found assembly '{assembly.Name}' multiple times: \n" + string.Join("\n", otherversions.Select(a => "- " + a.Location).Distinct())); } if (assembly.PluginTypes != null) { foreach (TypeData type in assembly.PluginTypes) { if (type.TypeAttributes.HasFlag(TypeAttributes.Interface) || type.TypeAttributes.HasFlag(TypeAttributes.Abstract)) continue; var supportedModels = GetSupportedModels(type).ToArray(); PluginFile plugin = new PluginFile { BaseType = string.Join(" | ", type.PluginTypes.Select(t => t.GetBestName())), Type = type.Name, Name = type.GetBestName(), Description = type.Display != null ? type.Display.Description : "", Groups = type.Display != null ? type.Display.Group : null, Collapsed = type.Display != null ? type.Display.Collapsed : false, Order = type.Display != null ? type.Display.Order : -10000, Browsable = type.IsBrowsable, SupportedModels = supportedModels, }; def.Plugins.Add(plugin); } } def.DependentAssemblies = assembly.References.ToList(); } else { // This error could be critical since assembly dependencies won't be found. log.Warning($"Could not load plugins for '{fullPath}'"); } } catch (BadImageFormatException) { // unable to load file. Ignore this error, it is probably not a .NET dll. } } } } } private static bool IsDotNetAssembly(string fullPath) { try { if (File.Exists(fullPath)) { AssemblyName testAssembly = AssemblyName.GetAssemblyName(fullPath); return true; } } catch (Exception) { } return false; } // TypeData has a PluginTypes property which contains the list of plugins // which it implements. ITypeData does not. Recursively iterate // the basetype hierarchy until we find a TypeData which has populated this property, // or until we find an ITypeData which inherits directly from ITapPlugin static ITypeData[] tdPluginTypes(ITypeData td) { if (td.BaseType == null || td.BaseType == td) return Array.Empty(); if (td.BaseType.IsA(typeof(ITapPlugin))) return new[] { td }; if (td is TypeData t) { // If PluginTypes is null, it could be a dynamic typedata whose // basetype has actual pluginTypes (seen with the Python plugin) if (t.PluginTypes == null) return tdPluginTypes(td.BaseType); // Otherwise if it does have plugin types, we can sim return t.PluginTypes.Cast().ToArray(); } return tdPluginTypes(td.BaseType); } private static void EnumerateAdditionalPlugins(PackageDef pkgDef) { string normalize(string s) => s.ToLowerInvariant().Replace("\\", "/").Replace("//", "/"); // Create a lookup of all plugin sources based on source file var sources = TypeData.GetDerivedTypes().Where(t => t.CanCreateInstance).Select(TypeData.GetTypeDataSource).Where(src => src.GetType() != typeof(AssemblyData)) .ToLookup(src => normalize(Path.GetFullPath(src.Location))); if (sources.Count == 0) return; var tapdir = ExecutorClient.ExeDir; var workDir = EngineSettings.StartupDir; foreach (var file in pkgDef.Files) { ITypeDataSource[] sourceList = Array.Empty(); if (!string.IsNullOrWhiteSpace(file.RelativeDestinationPath)) sourceList = sources[normalize(Path.Combine(tapdir, file.RelativeDestinationPath))].ToArray(); if (sourceList.Length == 0 && !string.IsNullOrWhiteSpace(file.FileName)) sourceList = sources[normalize(Path.Combine(tapdir, file.FileName))].ToArray(); // look in the working directory - tap package create is relative to that. if (sourceList.Length == 0 && !string.IsNullOrWhiteSpace(file.FileName)) sourceList = sources[normalize(Path.Combine(workDir, file.FileName))].ToArray(); if (sourceList.Length == 0) continue; // discinct by provider type (if a file has more than one plugin in it from the same provider, that provider is going to be in the list several times) sourceList = sourceList.GroupBy(src => src.GetType()).Select(grp => grp.First()).ToArray(); // Iterate all the sources that have generated plugins based on this file foreach (var source in sourceList) { // Add any relevant dependencies. Note that we only add dependencies on AssemblyData implementations, // It would probably be more correct if PackageFile.DependentAssemblies was called something like // PackageFile.PluginFileDependencies, and had a list of ITypeDataSource instead. var dependencies = source.References.ToArray(); file.DependentAssemblies.AddRange(dependencies.OfType()); file.DependentTypeDataSources.AddRange(dependencies.Where(x => !(x is AssemblyData))); string bestName(ITypeData t, DisplayAttribute display = null) { display ??= t.GetDisplayAttribute(); return display?.Name ?? t.Name.Split('.', '+').Last(); } foreach (var td in source.Types) { try { if (td.CanCreateInstance == false) continue; void addTypeDataDependencies(ITypeData td2) { if (td2 == null) return; var src = TypeData.GetTypeDataSource(td2); if (src.Location != null) { if (file.DependentTypeDataSources.Contains(src)) return; file.DependentTypeDataSources.Add(src); } addTypeDataDependencies(td2.BaseType); } addTypeDataDependencies(td.BaseType); var display = td.GetDisplayAttribute(); var pluginTypes = tdPluginTypes(td); var supportedModels = GetSupportedModels(td).ToArray(); var plug = new PluginFile() { BaseType = string.Join(" | ", pluginTypes.Select(t => bestName(t))), Type = td.Name, Name = bestName(td, display), Description = display?.Description ?? "", Groups = display?.Group, Collapsed = display?.Collapsed ?? false, Order = display?.Order ?? -10000, Browsable = td.GetAttribute()?.Browsable ?? true, SupportedModels = supportedModels, }; file.Plugins.Add(plug); } catch (Exception ex) { log.Error($"Failed to add plugins from '{td.Name}'"); log.Debug(ex); } } } // Remove duplicated dependencies (very unlikely) file.DependentAssemblies = file.DependentAssemblies.Distinct().ToList(); } } /// /// Load from an XML package definition file. /// This file is not expected to have info about the plugins in it, so this method will enumerate the plugins inside each dll by loading them. /// /// The Package Definition xml file. Usually named package.xml /// Directory used byt GitVersionCalculator to expand any $(GitVersion) macros in the XML file. /// public static PackageDef FromInputXml(string xmlFilePath, string projectDir) { try { var sw = Stopwatch.StartNew(); var evaluator = new PackageXmlPreprocessor(xmlFilePath, projectDir); var xmlDoc = evaluator.Evaluate(); var evaluated = Path.GetTempFileName(); xmlDoc.Save(evaluated); xmlFilePath = evaluated; log.Debug(sw, $"Package preprocessing completed."); } catch (Exception ex) { log.Warning(ex.Message); log.Debug($"Unexpected error while evaluating package xml. Continuing in spite of errors."); log.Debug(ex); } PackageDef.ValidateXml(xmlFilePath); var pkgDef = PackageDef.FromXml(xmlFilePath); if(pkgDef.Files.Any(f => f.HasCustomData() && f.HasCustomData())) throw new InvalidDataException("A file cannot specify and at the same time."); pkgDef.Files = expandGlobEntries(pkgDef.Files); var excludeAdd = pkgDef.Files.Where(file => file.IgnoredDependencies != null).SelectMany(file => file.IgnoredDependencies).Distinct().ToList(); List exceptions = new List(); foreach (PackageFile item in pkgDef.Files) { string fullPath = Path.GetFullPath(item.FileName); if (!File.Exists(fullPath)) { string fileName = Path.GetFileName(item.FileName); if (File.Exists(fileName) && item.SourcePath == null) { // this is to support building everything to the root folder. This way the developer does not have to specify SourcePath. log.Warning("Specified file '{0}' was not found, using file '{1}' as source instead. Consider setting SourcePath to remove this warning.", item.FileName,fileName); item.SourcePath = fileName; } else exceptions.Add(new FileNotFoundException("Missing file for package.", fullPath)); } } if (exceptions.Count > 0) throw new AggregateException("Missing files", exceptions); pkgDef.Date = DateTime.UtcNow; // Copy to output directory first foreach(var file in pkgDef.Files) { if (file.RelativeDestinationPath != file.FileName) try { var destPath = Path.GetFullPath(file.RelativeDestinationPath); if (!File.Exists(destPath)) { Directory.CreateDirectory(Path.GetDirectoryName(destPath)); ProgramHelper.FileCopy(file.FileName, destPath); } } catch { // Catching here. The files might be used by themselves } } var searcher = new PluginSearcher(PluginSearcher.Options.IncludeSameAssemblies); searcher.Search(Directory.GetCurrentDirectory()); List assemblies = searcher.Assemblies.ToList(); // Enumerate plugins if this has not already been done. if (!pkgDef.Files.SelectMany(pfd => pfd.Plugins).Any()) { // Enumerate plugin types from C# assemblies EnumeratePlugins(pkgDef, assemblies); // Enumerate plugin types from other ITypeDataSource implementations EnumerateAdditionalPlugins(pkgDef); } pkgDef.findDependenciesAndHandleOverrides(excludeAdd, assemblies); if (exceptions.Count > 0) throw new AggregateException("Conflicting dependencies", exceptions); return pkgDef; } internal static List expandGlobEntries(List fileEntries) { List newEntries = new List(); foreach (PackageFile fileEntry in fileEntries) { // Make SourcePath and RelativeDestinationPath Linux friendly if(fileEntry.SourcePath != null && fileEntry.SourcePath.Contains('\\')) { log.Info($"File path ({fileEntry.SourcePath}) in package definition contains `\\`, please consider replacing with `/`."); fileEntry.SourcePath = fileEntry.SourcePath.Replace('\\', '/'); } if(fileEntry.RelativeDestinationPath.Contains('\\')) { log.Info($"File path ({fileEntry.RelativeDestinationPath}) in package definition contains `\\`, please consider replacing with `/`."); fileEntry.RelativeDestinationPath = fileEntry.RelativeDestinationPath.Replace('\\', '/'); } if (fileEntry.FileName.Contains('*') || fileEntry.FileName.Contains('?')) { string[] segments = fileEntry.FileName.Split('/'); int fixedSegmentCount = 0; while (fixedSegmentCount < segments.Length) { if (segments[fixedSegmentCount].Contains('*')) break; if (segments[fixedSegmentCount].Contains('?')) break; fixedSegmentCount++; } string fixedDir = "."; if(fixedSegmentCount > 0) fixedDir = String.Join("/", segments.Take(fixedSegmentCount)); IEnumerable files = null; if (Directory.Exists(fixedDir)) { if (fixedSegmentCount == segments.Length - 1) files = Directory.GetFiles(fixedDir, segments.Last()); else { files = Directory.GetFiles(fixedDir, segments.Last(), SearchOption.AllDirectories) .Select(f => f.StartsWith(".") ? f.Substring(2) : f); // remove leading "./". GetFiles() adds these chars only when fixedDir="." and they confuse the Glob matching below. var options = new DotNet.Globbing.GlobOptions(); if(OperatingSystem.Current == OperatingSystem.Windows) options.Evaluation.CaseInsensitive = false; else options.Evaluation.CaseInsensitive = true; var globber = DotNet.Globbing.Glob.Parse(fileEntry.FileName, options); List matches = new List(); foreach (string file in files) { if (globber.IsMatch(file)) matches.Add(file); } files = matches; } } if (files != null && files.Any()) { newEntries.AddRange(files.Select(f => new PackageFile { RelativeDestinationPath = f.Replace('\\', '/'), LicenseRequired = fileEntry.LicenseRequired, IgnoredDependencies = fileEntry.IgnoredDependencies, // clone the list to ensure further modifications happens // a the expected place. CustomData = fileEntry.CustomData.ToList() })); } else { log.Warning("Glob pattern '{0}' did not match any files.", fileEntry.FileName); } } else { newEntries.Add(fileEntry); } } return newEntries; } internal static IEnumerable AssembliesOfferedBy(List packages, IEnumerable refs, bool recursive, PackageAssemblyCache offeredFiles) { var files = new HashSet(); var referenced = new HashSet(); var toLookat = new Stack(refs); while (toLookat.Any()) { var dep = toLookat.Pop(); if (referenced.Add(dep)) { var pkg = packages.Find(p => (p.Name == dep.Name) && dep.Version.IsCompatible(p.Version)); if (pkg != null) { if (recursive) pkg.Dependencies.ForEach(toLookat.Push); offeredFiles.GetPackageAssemblies(pkg).ToList().ForEach(f => files.Add(f)); } } } return files; } private static class AssemblyRefUtils { public static bool IsCompatibleReference(AssemblyData asm, AssemblyData reference) { return (asm.Name == reference.Name) && OpenTap.Utils.Compatible(asm.Version, reference.Version); } } internal class PackageAssemblyCache { readonly List brokenPackageNames = new List(); readonly List searchedFiles; readonly Memorizer> packageAssemblies; public PackageAssemblyCache(List searchedFiles) { this.searchedFiles = searchedFiles; packageAssemblies = new Memorizer>(getPackageAssemblies); } public IEnumerable GetPackageAssemblies(PackageDef pkgDef) => packageAssemblies[pkgDef]; List getPackageAssemblies(PackageDef pkgDef) { List output = new List(); foreach(var f in pkgDef.Files) { var asms = searchedFiles.Where(sf => PathUtils.AreEqual(f.FileName, sf.Location)) .Where(sf => IsDotNetAssembly(sf.Location)).ToList(); if (asms.Count == 0 && (Path.GetExtension(f.FileName).Equals(".dll", StringComparison.OrdinalIgnoreCase) || Path.GetExtension(f.FileName).Equals(".exe", StringComparison.OrdinalIgnoreCase))) { if (File.Exists(f.FileName) && !PathUtils.DecendsFromOpenTapIgnore(f.FileName)) { // If the pluginSearcher found assemblies that are located somewhere not expected by the package definition, the package might appear broken. // But if the file found by the pluginSearcher is the same as the one expected by the package definition we should not count it as broken. // This could cause a package to not be added as a dependencies. // E.g. when debugging and the OpenTAP.Cli.dll is both in the root build dir and in "Packages/OpenTAP" var asmsIdenticalFilename = searchedFiles.Where(sf => Path.GetFileName(f.FileName) == Path.GetFileName(sf.Location)); var asmsIdentical = asmsIdenticalFilename.Where(sf => PathUtils.CompareFiles(f.FileName, sf.Location)); output.AddRange(asmsIdentical); continue; } if (!brokenPackageNames.Contains(pkgDef.Name) && IsDotNetAssembly(f.FileName)) { brokenPackageNames.Add(pkgDef.Name); log.Warning($"Package '{pkgDef.Name}' is not installed correctly? Referenced file '{f.FileName}' was not found."); } } output.AddRange(asms); } return output; } public void Clear(PackageDef pkg) => packageAssemblies.Invalidate(pkg); } private static void VerifyDependenciesInstalled(this PackageDef pkg) { var currentInstallation = new Installation(Directory.GetCurrentDirectory()); if (!currentInstallation.IsInstallationFolder) // if there is no installation in the current folder look where tap is executed from currentInstallation = new Installation(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)); var installed = currentInstallation.GetPackages().Where(p => p.Name != pkg.Name).ToList(); // check versions of any hardcoded dependencies against what is currently installed foreach(PackageDependency dep in pkg.Dependencies) { var installedPackage = installed.FirstOrDefault(ip => ip.Name == dep.Name); if (installedPackage != null) { if (dep.Version == null) { dep.Version = new VersionSpecifier(installedPackage.Version, VersionMatchBehavior.Compatible); log.Info("A version was not specified for package dependency {0}. Using installed version ({1}).", dep.Name, dep.Version); } else { if (!dep.Version.IsCompatible(installedPackage.Version)) throw new ExitCodeException((int)PackageExitCodes.PackageDependencyError, $"Installed version of {dep.Name} ({installedPackage.Version}) is incompatible with dependency specified in package definition ({dep.Version})."); } } else { throw new ExitCodeException((int)PackageExitCodes.PackageDependencyError, $"Package dependency '{dep.Name}' specified in package definition is not installed. Please install a compatible version first."); } } } internal static void findDependenciesAndHandleOverrides(this PackageDef pkgDef, List excludeAdd, List searchedFiles) { // The dependencies of a package greatly influence dependency resolution. // This is because a package depending on e.g. 'OpenTAP' is treated as if it provides all the DLLs provided by OpenTAP. // This is useful because a dependency on e.g. `OpenTap.dll' can be resolved by adding a dependency on OpenTAP, // but it has the side effect of allowing a package to provide its version of any dll provided by OpenTAP. // As a workaround for this, we clear manual dependencies before resolution, and re-add them afterwards. var manualDependencies = pkgDef.Dependencies.ToArray(); pkgDef.Dependencies.Clear(); pkgDef.findDependencies(excludeAdd, searchedFiles); foreach (var md in manualDependencies) { var existing = pkgDef.Dependencies.FirstOrDefault(x => x.Name == md.Name); if (existing == null) { pkgDef.Dependencies.Add(md); } else { existing.Version = md.Version; } } pkgDef.VerifyDependenciesInstalled(); } internal static void findDependencies(this PackageDef pkg, List excludeAdd, List searchedFiles) { var searcher = new PackageAssemblyCache(searchedFiles); // First update the pre-entered dependencies bool foundNew = false; var notFound = new HashSet(); var currentInstallation = new Installation(Directory.GetCurrentDirectory()); if (!currentInstallation.IsInstallationFolder) // if there is no installation in the current folder look where tap is executed from currentInstallation = new Installation(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)); var installed = currentInstallation.GetPackages().Where(p => p.Name != pkg.Name).ToList(); { // add dependencies which are from different type data sources than AssemblyData (.NET). foreach (var file in pkg.Files) { foreach (var dep in file.DependentTypeDataSources) { var pkg2 = currentInstallation.FindPackageContainingFile(dep.Location); if (pkg2 == null) { log.Debug($"Dependency without a source package dependency: {dep.Name}, depender: {file.FileName}"); continue; } if (pkg.Dependencies.Any(dep => dep.Name == pkg2.Name)) { continue; } if (pkg.Name == pkg2.Name) // don't have the package depend on itself. This can happen if the package.xml getting created is already in a location as if it was installed (e.g. ./Packages//package.xml) continue; PackageDependency pd = new PackageDependency(pkg2.Name, new VersionSpecifier(pkg2.Version, VersionMatchBehavior.Compatible)); pkg.Dependencies.Add(pd); } } } // Find additional dependencies do { foundNew = false; // Find everything we already know about var offeredByDependencies = AssembliesOfferedBy(installed, pkg.Dependencies, false, searcher).ToList(); var offeredByThis = searcher.GetPackageAssemblies(pkg) .Where(f => f != null) .ToList(); var anyOffered = offeredByDependencies.Concat(offeredByThis).ToList(); // Find our dependencies and subtract the above two lists var dependentAssemblyNames = pkg.Files .SelectMany(fs => fs.DependentAssemblies) .Where(r => r.Name != "mscorlib") // Special case. We should not bundle the framework assemblies. .Where(r => !anyOffered.Any(of => AssemblyRefUtils.IsCompatibleReference(of, r))) .Distinct().Where(x => !excludeAdd.Contains(x.Name)).ToList(); // If there's anything left we start resolving if (dependentAssemblyNames.Any()) { // First look in installed packages var packageCandidates = new Dictionary(); foreach (var f in installed) { // Check if the package depends on the package being built. // Circular dependencies are not supported, so don't add it. // in the best of worlds, we'd recurse to find if there is a dependency // further up the tree, but that seems like a very unlikely situation, so // we skip that for now. if (f.Dependencies.Any(dep => dep.Name == pkg.Name)) continue; var candidateAsms = searcher.GetPackageAssemblies(f) .Where(asm => dependentAssemblyNames.Any(dep => (dep.Name == asm.Name))).ToList(); // Don't consider a package that only matches assemblies in the Dependencies subfolder candidateAsms.RemoveAll(asm => asm.Location.Contains("Dependencies")); // TODO: less lazy check for Dependencies subfolder would be good. if (candidateAsms.Count > 0) packageCandidates[f] = candidateAsms.Count; } // Look at the most promising candidate (i.e. the one containing most assemblies with the same names as things we need) PackageDef candidatePkg = packageCandidates.OrderByDescending(k => k.Value).FirstOrDefault().Key; if (candidatePkg != null) { foreach(AssemblyData candidateAsm in searcher.GetPackageAssemblies(candidatePkg)) { var requiredAsm = dependentAssemblyNames.FirstOrDefault(dep => dep.Name == candidateAsm.Name); if (requiredAsm != null) { if (candidateAsm.Version == null) throw new Exception($"Assembly {candidateAsm} version is null"); if (requiredAsm.Version == null) throw new Exception($"Assembly {requiredAsm} version is null"); if(OpenTap.Utils.Compatible(candidateAsm.Version, requiredAsm.Version)) { log.Info($"Satisfying assembly reference to {requiredAsm.Name} by adding dependency on package {candidatePkg.Name}"); if (candidateAsm.Version != requiredAsm.Version) { log.Warning($"Version of {requiredAsm.Name} in {candidatePkg.Name} is different from the version referenced in this package ({requiredAsm.Version} vs {candidateAsm.Version})."); log.Warning($"Consider changing your version of {requiredAsm.Name} to {candidateAsm.Version} to match that in {candidatePkg.Name}."); } foundNew = true; } else { var depender = pkg.Files.FirstOrDefault(f => f.DependentAssemblies.Contains(requiredAsm)); if (depender == null) log.Error($"This package require assembly {requiredAsm.Name} in version {requiredAsm.Version} while that assembly is already installed through package '{candidatePkg.Name}' in version {candidateAsm.Version}."); else log.Error($"{Path.GetFileName(depender.FileName)} in this package require assembly {requiredAsm.Name} in version {requiredAsm.Version} while that assembly is already installed through package '{candidatePkg.Name}' in version {candidateAsm.Version}."); //log.Error($"Please align the version of {requiredAsm.Name} to ensure interoperability with package '{candidate.Key.Name}' or uninstall that package."); throw new ExitCodeException((int)PackageExitCodes.AssemblyDependencyError, $"Please align the version of {requiredAsm.Name} ({candidateAsm.Version} vs {requiredAsm.Version}) to ensure interoperability with package '{candidatePkg.Name}' or uninstall that package."); } } } if (foundNew) { // Check if a circular dependency gets added into the package. // This should only happen in case of an error. Continuing will cause undefined behavior. var circularDependency = candidatePkg.Dependencies.FirstOrDefault(x => x.Name == pkg.Name); if (circularDependency != null) throw new Exception("A package cannot depend on another package that depends on it. Circular dependencies are not supported."); log.Info("Adding dependency on package '{0}' version {1}", candidatePkg.Name, candidatePkg.Version); PackageDependency pd = new PackageDependency(candidatePkg.Name, new VersionSpecifier(candidatePkg.Version, VersionMatchBehavior.Compatible)); pkg.Dependencies.Add(pd); } } else { // No installed package can offer any of the remaining referenced assemblies. // add them as payload in this package in the Dependencies subfolder foreach (var unknown in dependentAssemblyNames) { var foundAsms = searchedFiles.Where(asm => (asm.Name == unknown.Name) && OpenTap.Utils.Compatible(asm.Version, unknown.Version)).ToList(); var foundAsm = foundAsms.FirstOrDefault(); if (foundAsm != null) { AddFileDependencies(pkg, unknown, foundAsm); searcher.Clear(pkg); foundNew = true; } else if (!notFound.Contains(unknown.Name)) { log.Debug("'{0}' could not be found in any of {1} searched assemblies, or is already added.", unknown.Name, searchedFiles.Count); notFound.Add(unknown.Name); } } } } } while (foundNew); } private static void AddFileDependencies(PackageDef pkg, AssemblyData dependency, AssemblyData foundAsm) { var depender = pkg.Files.FirstOrDefault(f => f.DependentAssemblies.Contains(dependency)); var destPath = string.Format("Dependencies/{0}.{1}/{2}", Path.GetFileNameWithoutExtension(foundAsm.Location), foundAsm.Version.ToString(), Path.GetFileName(foundAsm.Location)); if (pkg.Files.Any(x => x.RelativeDestinationPath == destPath)) return;//throw new Exception("File already added to package: " + destPath); if (depender == null) log.Warning("Adding dependent assembly '{0}' to package. It was not found in any other packages.", Path.GetFileName(foundAsm.Location)); else log.Info($"'{Path.GetFileName(depender.FileName)}' depends on '{dependency.Name}' version '{dependency.Version}'. Adding dependency to package, it was not found in any other packages."); pkg.Files.Add(new PackageFile { SourcePath = foundAsm.Location, RelativeDestinationPath = destPath, DependentAssemblies = foundAsm.References.ToList() }); // Copy the file to the actual directory so we can rely on it actually existing where we say the package has it. if (!File.Exists(destPath)) { Directory.CreateDirectory(Path.GetDirectoryName(destPath)); ProgramHelper.FileCopy(foundAsm.Location, destPath); } } /// /// Creates a *.TapPackage file from the definition in this PackageDef. /// public static void CreatePackage(this PackageDef pkg, FileStream str) { foreach (PackageFile file in pkg.Files) { if (!File.Exists(file.FileName)) { log.Error("{0}: File '{1}' not found", pkg.Name, file.FileName); throw new InvalidDataException(); } } if (pkg.Files.Any(s => s.HasCustomData())) { bool resolved = true; foreach (var file in pkg.Files) { while (file.HasCustomData()) { MissingPackageData missingPackageData = file.GetCustomData().FirstOrDefault(); if (missingPackageData.TryResolve(out ICustomPackageData customPackageData)) { file.CustomData.Add(customPackageData); } else { resolved = false; log.Error($"Missing plugin to handle XML Element '{missingPackageData.XmlElement.Name.LocalName}' on file {file.FileName}. (Line {missingPackageData.GetLine()})"); } file.CustomData.Remove(missingPackageData); } } if (!resolved) throw new ArgumentException("Missing plugins to handle XML elements specified in input package.xml..."); } string tempDir = Path.GetTempPath() + Path.GetRandomFileName(); Directory.CreateDirectory(tempDir); log.Debug("Using temporary folder at '{0}'", tempDir); try { UpdateVersionInfo(tempDir, pkg.Files, pkg.Version); // License Inject // Obfuscate // Sign CustomPackageActionHelper.RunCustomActions(pkg, PackageActionStage.Create, new CustomPackageActionArgs(tempDir, false)); // Concat license required from all files. But only if the property has not manually been set. if (string.IsNullOrEmpty(pkg.LicenseRequired)) { var licenses = pkg.Files.Select(f => f.LicenseRequired).Where(l => string.IsNullOrWhiteSpace(l) == false).ToList(); pkg.LicenseRequired = string.Join(", ", licenses.Distinct().Select(l => LicenseBase.FormatFriendly(l, false)).ToList()); } log.Info("Creating OpenTAP package."); pkg.Compress(str, pkg.Files); // The filestream is opened with the 'DeleteOnClose' flag. If we throw here, // the file pointed to by the stream will be deleted. if (!VerifyArchiveIntegrity(str)) throw new ExitCodeException((int)PackageExitCodes.InvalidPackageArchive, "Failed to verify the integrity of the created package."); } finally { FileSystemHelper.DeleteDirectory(tempDir); } } [Display("SetAssemblyInfo")] public class SetAssemblyInfoData : ICustomPackageData { [XmlAttribute] public string Attributes { get; set; } internal string[] Features => Attributes.ToLower() .Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries) .Select(s => s.Trim()).Distinct().ToArray(); } private static void UpdateVersionInfo(string tempDir, List files, SemanticVersion version) { var timer = Stopwatch.StartNew(); var pdbMap = new Dictionary(); foreach (var file in files.GroupBy(f => Path.GetFileNameWithoutExtension(f.FileName))) { var symbols = file.ToArray().FirstOrDefault(f => Path.GetExtension(f.FileName).Equals(".pdb", StringComparison.OrdinalIgnoreCase)); if (symbols != null) pdbMap[file.Key] = symbols; } foreach (var file in files) { var data = file.GetCustomData().ToArray(); if (!data.Any(d => d.Features.Contains("version"))) continue; log.Debug(timer, "Updating version info for '{0}'", file.FileName); // Assume we can't open the file for writing (could be because we are trying to modify TPM or the engine), and copy to the same filename in a subdirectory var versionedOutput = Path.Combine(tempDir, "Versioned"); var origFilename = Path.GetFileName(file.FileName); var tempName = Path.Combine(versionedOutput, origFilename); int i = 1; while (File.Exists(tempName)) { tempName = Path.Combine(versionedOutput, origFilename + i.ToString()); i++; } Directory.CreateDirectory(Path.GetDirectoryName(tempName)); ProgramHelper.FileCopy(file.FileName, tempName); file.SourcePath = tempName; var includePdb = true; var basename = Path.GetFileNameWithoutExtension(file.SourcePath); if (pdbMap.TryGetValue(basename, out var pdbFile) && Path.GetFileName(pdbFile.FileName) is string symbolsFile && File.Exists(symbolsFile)) { var pdbTempName = Path.ChangeExtension(tempName, "pdb"); File.Copy(symbolsFile, pdbTempName); pdbFile.SourcePath = pdbTempName; } else { // The pdb file is not part of the package -- don't include it includePdb = false; } var fVersion = version; var fVersionShort = new Version(version.ToString(3)); SetAsmInfo.SetAsmInfo.SetInfo(file.FileName, fVersionShort, fVersionShort, fVersion, includePdb); file.RemoveCustomData(); } log.Info(timer,"Updated assembly version info using Mono method."); } private static bool VerifyArchiveIntegrity(FileStream fs) { var sw = Stopwatch.StartNew(); log.Debug($"Verifying package integrity..."); // Ensure the stream is fully written to the disk fs.Flush(); { // Verify that we can read the package definition try { PackageDef.FromPackage(fs.Name); } catch (Exception ex) { log.Error($"Error verifying package integrity."); log.Debug(ex); return false; } } log.Debug(sw, "Verified metadata integrity."); // Rewind the stream var files = new HashSet(); fs.Seek(0, SeekOrigin.Begin); // Verify that we can extract the archive without errors using (var zip = new ZipArchive(fs, ZipArchiveMode.Read, true)) { foreach (var part in zip.Entries) { try { if (!files.Add(part.FullName)) { log.Error($"Error verifying archive integrity. " + $"Archive contains a duplicate entry '{part.FullName}'." + $"Please retry the package creation. " + $"If the error persists, consider creating an issue."); return false; } // Read the stream to the end to verify it can be extracted part.Open().CopyTo(Stream.Null); } catch (InvalidDataException) { log.Error($"Error verifying package integrity. " + $"Archive entry '{part.FullName}' is corrupted. " + $"Please retry the package creation. " + $"If the error persists, consider creating an issue."); return false; } } } log.Debug(sw, $"Verified archive integrity."); return true; } /// /// Compresses the files to a zip package. /// private static void Compress(this PackageDef pkg, FileStream outStream, IEnumerable inputPaths) { using (var zip = new System.IO.Compression.ZipArchive(outStream, System.IO.Compression.ZipArchiveMode.Create, leaveOpen: true)) { foreach (PackageFile file in inputPaths) { var sw = Stopwatch.StartNew(); var relFileName = file.RelativeDestinationPath; var ZipPart = zip.CreateEntry(relFileName, System.IO.Compression.CompressionLevel.Optimal); ZipPart.FixUnixPermissions(); using (var instream = File.OpenRead(file.FileName)) { using (var outstream = ZipPart.Open()) { var compressTask = instream.CopyToAsync(outstream, 2048, TapThread.Current.AbortToken); ConsoleUtils.PrintProgressTillEnd(compressTask, "Compressing", () => instream.Position, () => instream.Length); } } log.Debug(sw, "Compressed '{0}'", file.FileName); } // add the metadata xml file: var metaPart = zip.CreateEntry(String.Join("/", PackageDef.PackageDefDirectory, pkg.Name, PackageDef.PackageDefFileName), System.IO.Compression.CompressionLevel.Optimal); metaPart.FixUnixPermissions(); using(var str = metaPart.Open()) pkg.SaveTo(str); } outStream.Flush(); } } }