// Copyright Keysight Technologies 2012-2019
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, you can obtain one at http://mozilla.org/MPL/2.0/.
using System.Diagnostics;
using System.Text;
using System.Text.RegularExpressions;
using LibGit2Sharp;
using System.Linq;
using System.Collections.Generic;
using System;
using System.IO;
using Tap.Shared;
namespace OpenTap.Package
{
///
/// Calculates the version number of a commit in a git repository
///
internal class GitVersionCalulator : IDisposable
{
private static readonly TraceSource log = Log.CreateSource("GitVersion");
private const string configFileName = ".gitversion";
private readonly LibGit2Sharp.Repository repo;
private readonly string RepoDir;
private class Config
{
public SemanticVersion Version { get => _version; set => _version = value; }
private SemanticVersion _version = new SemanticVersion(0, 0, 1, null, null);
/// version before it got parsed to a SemanticVersion. Possibly not valid.
public string RawVersion { get; set; }
///
/// Regex that runs against the FriendlyName of a branch to determine if it is a beta branch
/// (commits from this branch will get a "beta" prerelease identifier)
///
public List BetaBranchRegexes { get; private set; } = new List
{
new Regex("^integration$", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex("^develop$", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex("^dev$", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex("^master$", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex("^main$", RegexOptions.Compiled | RegexOptions.IgnoreCase)
};
public List BetaBranchPatterns { get; private set; } = new List
{
"^integration$",
"^master$",
"^develop$",
"^dev$",
"^main$"
};
///
/// Regex that runs against the FriendlyName of a branch to determine if it is a release branch
/// (commits from this branch will not get any prerelease identifier)
///
public Regex ReleaseBranchRegex { get; private set; } = new Regex("^release[0-9x]*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
public Regex ReleaseTagRegex { get; private set; } = new Regex(@"v\d+\.\d+\.\d+", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private int _maxBranchChars = 30;
///
/// Cap the length of the branch name to this many chars. This can be useful e.g. if the version number is used in a file name, which could otherwise become too long.
///
public int MaxBranchChars => _maxBranchChars;
public string ConfigFilePath;
private Config()
{
}
private static Regex configLineRegex = new Regex(@"^(?!#)(?.*?)\s*=\s*(?.*)", RegexOptions.Compiled);
public static Config ParseConfig(Stream str, string configFilePath = configFileName)
{
Config cfg = new Config();
if (str == null)
return cfg;
cfg.ConfigFilePath = configFilePath;
bool isBetaBranchSet = false;
using (var reader = new StreamReader(str, Encoding.UTF8))
{
String line;
while ((line = reader.ReadLine()) != null)
{
var m = configLineRegex.Match(line);
if (m.Success)
{
string val = m.Groups["value"].Value;
switch (m.Groups["key"].Value.ToLower())
{
case "beta branch":
if (!isBetaBranchSet)
{
cfg.BetaBranchRegexes = new List();
cfg.BetaBranchPatterns = new List();
}
isBetaBranchSet = true;
cfg.BetaBranchRegexes.Add(new Regex(val, RegexOptions.IgnoreCase));
cfg.BetaBranchPatterns.Add(val);
break;
case "release branch":
cfg.ReleaseBranchRegex = new Regex(val, RegexOptions.IgnoreCase);
break;
case "release tag":
cfg.ReleaseTagRegex = new Regex(val, RegexOptions.IgnoreCase);
break;
case "max branch chars":
int.TryParse(val, out cfg._maxBranchChars);
break;
case "version":
cfg.RawVersion = val;
SemanticVersion.TryParse(val, out cfg._version);
break;
}
}
}
}
return cfg;
}
}
private const string GIT_HASH = "b7bad55";
void ensureLibgit2Present()
{
string libgit2name;
if (OperatingSystem.Current == OperatingSystem.Windows)
libgit2name = $"git2-{GIT_HASH}.dll";
else if (OperatingSystem.Current == OperatingSystem.Linux)
libgit2name = $"libgit2-{GIT_HASH}.so";
else if (OperatingSystem.Current == OperatingSystem.MacOS)
libgit2name = $"libgit2-{GIT_HASH}.dylib";
else
{
log.Error($"Unsupported platform.");
return;
}
var requiredFile = Path.Combine(PathUtils.OpenTapDir, libgit2name);
if (File.Exists(requiredFile))
return;
string sourceFile = Path.Combine(PathUtils.OpenTapDir, "Dependencies/LibGit2Sharp.0.27.0.0/", libgit2name);
if (OperatingSystem.Current == OperatingSystem.Windows)
sourceFile += $".{(Environment.Is64BitProcess ? CpuArchitecture.x64 : CpuArchitecture.x86)}";
if (OperatingSystem.Current == OperatingSystem.MacOS)
sourceFile += $".{MacOsArchitecture.Current.Architecture}";
if (OperatingSystem.Current == OperatingSystem.Linux)
sourceFile += $".{LinuxArchitecture.Current.Architecture}";
try
{
File.Copy(sourceFile, requiredFile, true);
}
catch (Exception e)
{
if (OperatingSystem.Current == OperatingSystem.Windows)
{
var opentapArch = Installation.Current.GetOpenTapPackage()?.Architecture;
var processArch = Environment.Is64BitProcess ? CpuArchitecture.x64 : CpuArchitecture.x86;
if (opentapArch != processArch)
throw new PlatformNotSupportedException($"Unable to find the correct 'libgit2-{GIT_HASH}' because the process architecture '{processArch}' does not match the installed OpenTAP architecture '{opentapArch}'", e);
}
throw new PlatformNotSupportedException($"Unable to copy 'libgit2-{GIT_HASH}': {e.Message}.", e);
}
}
///
/// Instanciates a new to work on a specified git repository.
///
/// Path pointing to a directory inside the git repository to use.
public GitVersionCalulator(string repositoryDir)
{
repositoryDir = Path.GetFullPath(repositoryDir);
RepoDir = repositoryDir;
while (!Directory.Exists(Path.Combine(repositoryDir, ".git")))
{
repositoryDir = Path.GetDirectoryName(repositoryDir);
if (repositoryDir == null)
throw new ArgumentException("Directory is not a git repository.", "repositoryDir");
}
RepoDir = RepoDir.Substring(repositoryDir.Length);
ensureLibgit2Present();
repo = new LibGit2Sharp.Repository(repositoryDir);
}
public void Dispose()
{
if (repo != null)
repo.Dispose();
}
/// Keeps iterating until a valid version is read.
SemanticVersion getLatestReadableVersion(Commit c)
{
while (c != null)
{
var cfg = readConfig(c);
if (cfg.Version != null) return cfg.Version;
c = getLatestConfigVersionChange(c.Parents.FirstOrDefault());
}
// no version was found.
return null;
}
static IEnumerable GetAllConfigFilesInTree(Tree t)
{
foreach (TreeEntry te in t)
{
if (te.Target is Tree subtree)
{
foreach (var match in GetAllConfigFilesInTree(subtree))
yield return match;
}
else if (te.Name == configFileName)
{
yield return te;
}
}
}
Config readConfig(Commit c)
{
var cfgFiles = GetAllConfigFilesInTree(c?.Tree).ToList();
string repositoryDir = RepoDir.TrimStart('/','\\');
TreeEntry cfg = null;
while (cfg == null)
{
var dir = Path.Combine(repositoryDir ?? "", configFileName).Replace('\\','/');
cfg = cfgFiles.FirstOrDefault(c => c.Path == dir);
if (String.IsNullOrEmpty(repositoryDir))
break;
repositoryDir = Path.GetDirectoryName(repositoryDir);
}
Blob configBlob = cfg?.Target as Blob;
return Config.ParseConfig(configBlob?.GetContentStream(), cfg?.Path);
}
Config ParseConfig(Commit c)
{
var cfg = readConfig(c);
if (cfg.Version == null)
{
log.Error("Unable to parse version specification {0}. It is not a valid semantic version.", cfg.RawVersion);
var ver = getLatestReadableVersion(c.Parents.FirstOrDefault());
if (ver != null)
{
log.Warning("Using previous {0} as version instead.", ver);
cfg.Version = ver;
}
}
return cfg;
}
private Commit getLatestConfigVersionChange(Commit c)
{
if (c.Parents.Any() == false)
return c; // 'c' is the first commit in the repo. There was never any change.
// find all changes in the file (for some reason that sometimes returns an empty list)
//var fileLog = repo.Commits.QueryBy(configFileName, new CommitFilter() { IncludeReachableFrom = c, SortBy = CommitSortStrategies.Topological, FirstParentOnly = false });
//... go on to iterate through filelog...
// Instead, just walk all commits comparing the version in the .gitversion file to the one in the previous commit
Config currentCfg = readConfig(c);
while (true)
{
Commit parent = c.Parents.FirstOrDefault(); // first parent only, we are only interested in when the file changes on the beta branch
if (parent == null)
{
// we got to the very first commit in this repo without seeing any changes in the gitversion
// this might be because there is no .gitversion file, or just because the content of the file is the same as the default values.
// in both cases, we should treat this commit (the initial commit) as the LatestConfigVersionChange
return c;
}
Config parentCfg = readConfig(parent);
if (currentCfg.Version != null && (parentCfg.Version == null || currentCfg.Version.CompareTo(parentCfg.Version) > 0))
{
// the version number was bumped
return c;
}
c = parent;
currentCfg = parentCfg;
}
}
///
/// Calculates the version number of the current HEAD of the git repository
///
public SemanticVersion GetVersion()
{
if (!repo.Commits.Any())
return new SemanticVersion(0, 0, 0, null, null);
return GetVersion(repo.Head.Tip);
}
///
/// Calculates the version number of a specific commit in the git repository
///
public SemanticVersion GetVersion(string sha)
{
Commit commit = repo.Lookup(sha);
if (commit == null)
throw new ArgumentException($"The commit with reference {sha} does not exist in the repository.");
return GetVersion(commit);
}
///
/// Calculates the version number of a specific commit in the git repository
///
public SemanticVersion GetVersion(Commit targetCommit)
{
if (repo.Lookup(targetCommit.Sha) == null)
throw new ArgumentException($"The commit with hash {targetCommit} does not exist the in repository.");
if(!GetAllConfigFilesInTree(targetCommit.Tree).Any())
{
log.Warning("Did not find any .gitversion file.");
}
Config cfg = ParseConfig(targetCommit);
if (cfg.ConfigFilePath != configFileName)
log.Debug("Using configuration from {0}", cfg.ConfigFilePath);
Branch defaultBranch = getBetaBranch(cfg);
string branchName = guessBranchName(cfg,targetCommit,defaultBranch);
string preRelease = "alpha";
if (branchName == defaultBranch.GetShortName())
preRelease = "beta";
if (cfg.ReleaseBranchRegex.IsMatch(branchName))
preRelease = "rc";
Tag releaseTag = getReleaseTag(cfg, targetCommit);
if (releaseTag != null)
preRelease = null;
string metadata = targetCommit.Sha.Substring(0, 8);
if (preRelease == "alpha")
{
if (branchName == "(no branch)")
branchName = "NONE"; // '(' and ' ' are not allowed in semver
else
branchName = Regex.Replace(branchName, "[^a-zA-Z0-9-]", "-"); // replace any chars that is not valid semver with '-'
if (branchName.Length > cfg.MaxBranchChars)
branchName = branchName.Remove(cfg.MaxBranchChars);
metadata += "." + branchName;
}
if (!String.IsNullOrEmpty(preRelease))
{
// The version calculation is slightly different for RC versions
// For an RC, we want to count merge commits as a single commit
// For other branches, we want to count the literal number of commits
// Historically, we have counted merge commits as single commits for all branches,
// but this causes issues in scenarios where merge commits are fast-forwarded onto e.g. the main branch.
// See here: https://github.com/opentap/opentap/pull/1384
// And here: https://github.com/opentap/opentap/issues/1321#issuecomment-1895749385
bool isRc = preRelease.StartsWith("rc", StringComparison.OrdinalIgnoreCase);
Commit cfgCommit = getLatestConfigVersionChange(targetCommit);
Commit commonAncestor = findFirstCommonAncestor(defaultBranch, targetCommit);
int commitsFromDefaultBranch = countCommitsBetween(commonAncestor, targetCommit, firstParentOnly: isRc);
log.Debug("Found {0} commits since branchout from beta branch in commit {1}.", commitsFromDefaultBranch, commonAncestor.Sha.Substring(0, 8));
int commitsSinceVersionUpdate = countCommitsBetween(cfgCommit, targetCommit, firstParentOnly: isRc) + 1;
log.Debug("Found {0} commits since last version bump in commit {1}.", commitsSinceVersionUpdate, cfgCommit.Sha.Substring(0, 8));
int alphaVersion = Math.Min(commitsFromDefaultBranch, commitsSinceVersionUpdate);
if (isRc == false)
{
int betaVersion = countCommitsBetween(cfgCommit, commonAncestor, false) + 1;
if (betaVersion > 0)
{
preRelease += "." + betaVersion;
}
}
if (alphaVersion > 0)
{
preRelease += "." + alphaVersion;
}
}
if (cfg.Version == null) return new SemanticVersion(0, 0, 0, preRelease, metadata);
return new SemanticVersion(cfg.Version.Major,cfg.Version.Minor,cfg.Version.Patch,preRelease,metadata);
}
private Tag getReleaseTag(Config cfg, Commit c)
{
foreach (Tag t in repo.Tags)
{
if (t.IsAnnotated &&
t.Target.Peel() == c &&
cfg.ReleaseTagRegex.IsMatch(t.FriendlyName))
{
return t;
}
}
return null;
}
///
/// Find the first (youngest) commit that is reachable from two specified places
///
private Commit findFirstCommonAncestor(Branch b1, Commit target)
{
// This fixes gitversion calculation in scenarios where the local revision of
// a checked out branch is behind the origin branch. If the local revision is fully merged
// in the remote tracking branch, we base our calculation on the remote branch instead.
// Otherwise, if the local branch contains commits that are *not* merged in the remote, we base the calculation on that.
// This should make the gitversion calculation work as expected after `git fetch --all`.
if (b1.TrackedBranch != null)
{
// Check if any local commits are unreachable from the tracking branch
var commitsMissingFromUpstream = (IQueryableCommitLog)repo.Commits.QueryBy(new CommitFilter() { IncludeReachableFrom = b1.Tip, ExcludeReachableFrom = b1.TrackedBranch.Tip});
if (commitsMissingFromUpstream.Any())
{
throw new Exception(
$"The local branch '{b1.GetShortName()}' contains commits missing from the tracked upstream branch.\n" +
$"This can cause unexpected mismatching version numbers. Please align '{b1.GetShortName()}' with its upstream.");
}
b1 = b1.TrackedBranch;
}
Commit b1Commit = b1.Tip;
while (b1Commit != null)
{
if (b1Commit.Sha == target.Sha)
return target; // target is a directly on the b1 branch
b1Commit = b1Commit.Parents.FirstOrDefault();
}
log.Debug($"Common ancestor of {b1.Tip} and {target} is not on the same branch.");
HashSet targetHistory = repo.Commits.QueryBy(new CommitFilter() { IncludeReachableFrom = target }).ToHashSet();
Commit firstCommon = b1.Commits.FirstOrDefault(c => targetHistory.Contains(c)); // same as repo.ObjectDatabase.FindMergeBase(b1.Tip, target); but faster on average
// if this branch is being used for several releases (merged to several times, one for each release)
// we will need to check against older releases as well as target might already exist on the tip of
// this release branch (i.e. it could have been merged there "in the future").
b1Commit = b1.Tip;
while (firstCommon == target) // this can happen if target is later merged into b1
{
if (targetHistory.Contains(b1Commit))
{
// We have reached past the begining of the release branch. There is no point in going further
firstCommon = null;
break;
}
b1Commit = b1Commit.Parents.FirstOrDefault();
var releaseCommits = (IQueryableCommitLog)repo.Commits.QueryBy(new CommitFilter() { SortBy = CommitSortStrategies.Topological, IncludeReachableFrom = b1Commit });
firstCommon = releaseCommits.FirstOrDefault(c => targetHistory.Contains(c));
}
return firstCommon;
}
private int countCommitsBetween(object tag, object now, bool firstParentOnly = false)
{
var filter = new CommitFilter()
{
SortBy = CommitSortStrategies.Reverse | CommitSortStrategies.Time,
ExcludeReachableFrom = tag,
IncludeReachableFrom = now,
FirstParentOnly = firstParentOnly
};
return repo.Commits.QueryBy(filter).Count();
}
private Branch getBetaBranch(Config cfg)
{
// Try to find the HEAD of the
foreach (var remote in repo.Network.Remotes)
{
string expectedDefaultRefName = $"refs/remotes/{remote.Name}/HEAD";
var defaultRef = repo.Refs.FirstOrDefault(r => r.CanonicalName == expectedDefaultRefName) as SymbolicReference;
if (defaultRef != null)
{
// be careful to return the remote branch instead of any local one. On build runners the local branch might be behind, as they usually just checkout a sha not the actual branch
var branch = repo.Branches.FirstOrDefault(b => b.CanonicalName == defaultRef.TargetIdentifier);
if (branch != null)
{
log.Debug("Determined beta branch to be '{0}' by looking at the HEAD of the remote '{1}'.",
branch.GetShortName(), remote.Name);
return branch;
}
}
}
// For each regex from the config, try to find a branch that matches.
Branch defaultBranch = cfg.BetaBranchRegexes.Select(rx => repo.Branches.FirstOrDefault(b => rx.IsMatch(b.GetShortName()))).FirstOrDefault(b => b != null);
if (defaultBranch == null)
{
StringBuilder error = new StringBuilder("Unable to determine the default branch. No branch matching ");
error.Append(String.Join(", ", cfg.BetaBranchPatterns.SkipLastN(1).Select(p => $"'{p}'")));
if (cfg.BetaBranchPatterns.Count > 1)
error.Append($" or ");
error.Append($"'{cfg.BetaBranchPatterns.Last()}' could be found. Searched {repo.Branches.Count()} branches.");
log.Error(error.ToString());
log.Debug("Branches:");
int c = 0;
StringBuilder line = new StringBuilder();
foreach (Branch item in repo.Branches.OrderBy(b => b.FriendlyName))
{
string bName = item.FriendlyName;
if (bName.Length > 25)
bName = bName.Substring(0, 25 - 3) + "...";
line.AppendFormat("{0,-26}", bName);
c++;
if (c == 4)
{
log.Debug(line.ToString());
c = 0;
line.Clear();
}
}
if (line.Length > 0)
log.Debug(line.ToString());
throw new NotSupportedException(error.ToString());
}
log.Debug("Determined beta branch to be '{0}' using regular expression match.", defaultBranch.GetShortName());
return defaultBranch;
}
///
/// Try to find the name of the branch a commit was originally created on. Logs warnings if not sure.
///
private string guessBranchName(Config cfg,Commit commit, Branch defaultBranch)
{
// is this the tip of the current branch
if (repo.Head.Tip.Sha == commit.Sha && repo.Info.IsHeadDetached == false)
return repo.Head.GetShortName();
// is the commit directly on the default branch?
var commitsOnDefault = repo.Commits.QueryBy(new CommitFilter { IncludeReachableFrom = defaultBranch, FirstParentOnly = true });
var shasOnDefault = commitsOnDefault.Select(c => c.Sha).ToHashSet();
if (shasOnDefault.Contains(commit.Sha))
{
// is this also the tip of a release branch, then pick that instead
var releaseBranch = repo.Branches.Where(r => cfg.ReleaseBranchRegex.IsMatch(r.GetShortName()) && r.Tip.Sha == commit.Sha).FirstOrDefault();
if (releaseBranch != null)
{
return releaseBranch.GetShortName();
}
return defaultBranch.GetShortName();
}
// is the commit the tip of any branches
var tipMatches = repo.Branches.Where(r => r.Tip.Sha == commit.Sha).Where(r => !r.FriendlyName.EndsWith("HEAD"));
if (tipMatches.Any())
{
if (tipMatches.Count() == 1)
return tipMatches.First().GetShortName();
var releaseBranch = tipMatches.FirstOrDefault(b => cfg.ReleaseBranchRegex.IsMatch(b.FriendlyName));
if (releaseBranch != null)
return releaseBranch.GetShortName();
if (tipMatches.Contains(defaultBranch))
return defaultBranch.GetShortName();
log.Warning("This commit is the tip of several branches, picking one. ({0})",String.Join(", ", tipMatches.Select(b => b.FriendlyName)));
return tipMatches.First().GetShortName();
}
// is the commit on the default branch indirectly through a merge commit
foreach (Commit onDefault in commitsOnDefault)
{
if (onDefault.Parents.Count() > 1)
{
Commit commitOnBranch = onDefault.Parents.Last();
while (!shasOnDefault.Contains(commitOnBranch.Sha))
{
if (commitOnBranch.Sha == commit.Sha)
{
var m = Regex.Match(onDefault.MessageShort, "Merge branch '([^']*)'");
if (m.Success)
{
string branchName = m.Groups[m.Groups.Count - 1].Value;
if (branchName.StartsWith("origin/"))
branchName = branchName.Substring(7);
return branchName;
}
else
{
log.Warning("Unable to determine old branch name. The branch has probably been deleted.");
return "DELETED";
}
}
commitOnBranch = commitOnBranch.Parents.First();
}
}
}
// is the commit on a branch that has not yet been merged to the default branch?
Stopwatch timer = Stopwatch.StartNew();
List candidates = new List();
foreach (var branch in repo.Branches)
{
Commit commitOnBranch = branch.Tip;
while (!shasOnDefault.Contains(commitOnBranch.Sha))
{
if (commitOnBranch.Sha == commit.Sha)
{
candidates.Add(branch);
break;
}
if (!commitOnBranch.Parents.Any())
break;
commitOnBranch = commitOnBranch.Parents.First();
}
}
TimeSpan dur = timer.Elapsed;
if (!candidates.Any())
{
log.Warning("Unable to determine branch name.");
return "ERROR";
}
if (candidates.Count == 1)
return candidates.First().GetShortName();
if(candidates.Select(b => b.GetShortName()).Distinct().Count() == 1) // all candicates have the same name (e.g. on is a local branch and one is the remote of that same branch)
return candidates.First().GetShortName();
log.Warning("Several possible branch names found. Picking one.");
// pick the first candidate that is has a merge commit as the child of commit
try
{
for (int b = 0; b < candidates.Count; b++)
{
var commitsOnB = repo.Commits.QueryBy(new CommitFilter { IncludeReachableFrom = candidates[b], ExcludeReachableFrom = commit, FirstParentOnly = true });
if (commitsOnB.Last().Parents.Count() == 2 && commitsOnB.Last().Parents.First() == commit)
return candidates[b].GetShortName();
}
}
catch
{
}
// TODO: we need some better logic to pick the right candidate here
//var selected = candidates.Select(b => (b, branchCountFromDefault(b, shasOnDefault))).ToList();
return candidates.First().GetShortName();
}
}
internal static class libGit2Helpers
{
public static string GetShortName(this Branch b)
{
if (b.FriendlyName.StartsWith("origin/"))
return b.FriendlyName.Substring(7);
return b.FriendlyName;
}
}
}