Skip to content

Commit

Permalink
SLCORE-1133: Rely on native Git if possible for blaming (#1228)
Browse files Browse the repository at this point in the history
---------

Co-authored-by: Nicolas Quinquenel <nicolas.quinquenel@sonarsource.com>
Co-authored-by: Kirill Knize <kirill.knize@sonarsource.com>
  • Loading branch information
3 people authored Feb 18, 2025
1 parent 67fe16c commit d366685
Show file tree
Hide file tree
Showing 10 changed files with 468 additions and 14 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
/*
* SonarLint Core - Commons
* Copyright (C) 2016-2025 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonarsource.sonarlint.core.commons.util.git;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
import java.nio.file.Path;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Date;
import java.util.regex.Pattern;
import org.sonar.scm.git.blame.BlameResult;
import org.sonarsource.sonarlint.core.commons.SonarLintBlameResult;

public class BlameParser {

private static final String FILENAME = "filename ";
private static final String AUTHOR_MAIL = "author-mail ";
private static final String COMMITTER_TIME = "committer-time ";
// if this text change between different git versions it will break the implementation
private static final String NOT_COMMITTED = "<not.committed.yet>";
private BlameParser() {
// Utility class
}

public static SonarLintBlameResult parseBlameOutput(String blameOutput, String currentFilePath, Path projectBaseDir) throws IOException {
var blameResult = new BlameResult();
var currentFileBlame = new BlameResult.FileBlame(currentFilePath, countCommitterTimeOccurrences(blameOutput));
blameResult.getFileBlameByPath().put(currentFilePath, currentFileBlame);
var fileSections = blameOutput.split(FILENAME);
var currentLineNumber = 0;

for (var fileSection : fileSections) {
if (fileSection.isBlank()) {
continue;
}

try (var reader = new BufferedReader(new StringReader(fileSection))) {
String line;
while ((line = reader.readLine()) != null) {
if (line.startsWith(AUTHOR_MAIL)) {
var authorEmail = line.substring(AUTHOR_MAIL.length());
currentFileBlame.getAuthorEmails()[currentLineNumber] = authorEmail;
}
if (line.startsWith(COMMITTER_TIME) && !currentFileBlame.getAuthorEmails()[currentLineNumber].equals(NOT_COMMITTED)) {
var committerTime = line.substring(COMMITTER_TIME.length());
var commitDate = Date.from(Instant.ofEpochSecond(Long.parseLong(committerTime)).truncatedTo(ChronoUnit.SECONDS));
currentFileBlame.getCommitDates()[currentLineNumber] = commitDate;
}
}
}
currentLineNumber++;
}

return new SonarLintBlameResult(blameResult, projectBaseDir);
}

public static int countCommitterTimeOccurrences(String blameOutput) {
var pattern = Pattern.compile("^" + COMMITTER_TIME, Pattern.MULTILINE);
var matcher = pattern.matcher(blameOutput);
var count = 0;
while (matcher.find()) {
count++;
}
return count;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,36 +26,49 @@
import java.io.InputStream;
import java.net.URI;
import java.nio.file.Path;
import java.util.LinkedList;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
import java.util.function.UnaryOperator;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nullable;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.SystemUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.diff.RawTextComparator;
import org.eclipse.jgit.ignore.IgnoreNode;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.RepositoryBuilder;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.sonar.scm.git.blame.RepositoryBlameCommand;
import org.sonarsource.sonarlint.core.commons.SonarLintBlameResult;
import org.sonarsource.sonarlint.core.commons.SonarLintGitIgnore;
import org.sonarsource.sonarlint.core.commons.Version;
import org.sonarsource.sonarlint.core.commons.log.SonarLintLogger;

import static java.util.Optional.ofNullable;
import static org.eclipse.jgit.lib.Constants.GITIGNORE_FILENAME;
import static org.sonarsource.sonarlint.core.commons.util.git.BlameParser.parseBlameOutput;
import static org.sonarsource.sonarlint.core.commons.util.git.WinGitUtils.locateGitOnWindows;

public class GitUtils {

private static final SonarLintLogger LOG = SonarLintLogger.get();
private static final String MINIMUM_REQUIRED_GIT_VERSION = "2.24.0";
private static final Pattern whitespaceRegex = Pattern.compile("\\s+");
private static final Pattern semanticVersionDelimiter = Pattern.compile("\\.");

// So we only have to make the expensive call once (or at most twice) to get the native Git executable!
private static boolean checkedForNativeGitExecutable = false;
private static String nativeGitExecutable = null;

private GitUtils() {
// Utility class
Expand Down Expand Up @@ -84,6 +97,21 @@ public static List<URI> getVSCChangedFiles(@Nullable Path baseDir) {
}
}

public static SonarLintBlameResult getBlameResult(Path projectBaseDir, Set<Path> projectBaseRelativeFilePaths, @Nullable UnaryOperator<String> fileContentProvider) {
return getBlameResult(projectBaseDir, projectBaseRelativeFilePaths, fileContentProvider, GitUtils::checkIfEnabled);
}

public static SonarLintBlameResult getBlameResult(Path projectBaseDir, Set<Path> projectBaseRelativeFilePaths, @Nullable UnaryOperator<String> fileContentProvider,
Predicate<Path> isEnabled) {
if (isEnabled.test(projectBaseDir)) {
LOG.debug("Using native git blame");
return blameFromNativeCommand(projectBaseDir, projectBaseRelativeFilePaths);
} else {
LOG.debug("Falling back to JGit git blame");
return blameWithFilesGitCommand(projectBaseDir, projectBaseRelativeFilePaths, fileContentProvider);
}
}

public static SonarLintBlameResult blameWithFilesGitCommand(Path projectBaseDir, Set<Path> projectBaseRelativeFilePaths, @Nullable UnaryOperator<String> fileContentProvider) {
var gitRepo = buildGitRepository(projectBaseDir);

Expand All @@ -110,6 +138,90 @@ public static SonarLintBlameResult blameWithFilesGitCommand(Path projectBaseDir,
}
}

/**
* Get the native Git executable by checking for the version of both `git` and `git.exe`. We cache this information
* to run these expensive processes more than once (or twice in case of Windows).
*/
private static String getNativeGitExecutable() {
if (!checkedForNativeGitExecutable) {
try {
var executable = getGitExecutable();
var process = new ProcessBuilder(executable, "--version").start();
var exitCode = process.waitFor();
if (exitCode == 0) {
nativeGitExecutable = executable;
}
} catch (IOException e) {
LOG.debug("Checking for native Git executable failed", e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
checkedForNativeGitExecutable = true;
}
return nativeGitExecutable;
}

private static String getGitExecutable() throws IOException {
return SystemUtils.IS_OS_WINDOWS ? locateGitOnWindows() : "git";
}

private static String executeGitCommand(Path workingDir, String... command) throws IOException {
var commandResult = new LinkedList<String>();
new ProcessWrapperFactory()
.create(workingDir, commandResult::add, command)
.execute();
return String.join(System.lineSeparator(), commandResult);
}

public static SonarLintBlameResult blameFromNativeCommand(Path projectBaseDir, Set<Path> projectBaseRelativeFilePaths) {
var nativeExecutable = getNativeGitExecutable();
if (nativeExecutable != null) {
for (var relativeFilePath : projectBaseRelativeFilePaths) {
try {
return parseBlameOutput(executeGitCommand(projectBaseDir,
nativeExecutable, "blame", projectBaseDir.resolve(relativeFilePath).toString(), "--line-porcelain", "--encoding=UTF-8"),
projectBaseDir.resolve(relativeFilePath).toString().replace("\\", "/"), projectBaseDir);
} catch (IOException e) {
throw new IllegalStateException("Failed to blame repository files", e);
}
}
}
throw new IllegalStateException("There is no native Git available");
}

public static boolean checkIfEnabled(Path projectBaseDir) {
var nativeExecutable = getNativeGitExecutable();
if (nativeExecutable == null) {
return false;
}
try {
var output = executeGitCommand(projectBaseDir, nativeExecutable, "--version");
return output.contains("git version") && isCompatibleGitVersion(output);
} catch (IOException e) {
return false;
}
}

private static boolean isCompatibleGitVersion(String gitVersionCommandOutput) {
// Due to the danger of argument injection on git blame the use of `--end-of-options` flag is necessary
// The flag is available only on git versions >= 2.24.0
var gitVersion = whitespaceRegex
.splitAsStream(gitVersionCommandOutput)
.skip(2)
.findFirst()
.orElse("");

var formattedGitVersion = formatGitSemanticVersion(gitVersion);
return Version.create(formattedGitVersion).compareToIgnoreQualifier(Version.create(MINIMUM_REQUIRED_GIT_VERSION)) >= 0;
}

private static String formatGitSemanticVersion(String version) {
return semanticVersionDelimiter
.splitAsStream(version)
.takeWhile(NumberUtils::isCreatable)
.collect(Collectors.joining("."));
}

private static Path getRelativePath(Repository gitRepo, Path projectBaseDir) {
var repoDir = gitRepo.isBare() ? gitRepo.getDirectory() : gitRepo.getWorkTree();
return repoDir.toPath().relativize(projectBaseDir);
Expand All @@ -124,7 +236,7 @@ public static Repository buildGitRepository(Path basedir) {
}

var repository = repositoryBuilder.build();
try (ObjectReader objReader = repository.getObjectDatabase().newReader()) {
try (var objReader = repository.getObjectDatabase().newReader()) {
// SONARSCGIT-2 Force initialization of shallow commits to avoid later concurrent modification issue
objReader.getShallowCommits();
return repository;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
/*
* SonarLint Core - Commons
* Copyright (C) 2016-2025 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonarsource.sonarlint.core.commons.util.git;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import java.util.Scanner;
import java.util.function.Consumer;
import javax.annotation.Nullable;
import org.sonarsource.sonarlint.core.commons.log.SonarLintLogger;

import static java.lang.String.format;
import static java.lang.String.join;
import static java.nio.charset.StandardCharsets.UTF_8;

public class ProcessWrapperFactory {
private static final SonarLintLogger LOG = SonarLintLogger.get();

public ProcessWrapperFactory() {
// nothing to do
}

public ProcessWrapper create(@Nullable Path baseDir, Consumer<String> stdOutLineConsumer, String... command) {
return new ProcessWrapper(baseDir, stdOutLineConsumer, Map.of(), command);
}

public ProcessWrapper create(@Nullable Path baseDir, Consumer<String> stdOutLineConsumer, Map<String, String> envVariables, String... command) {
return new ProcessWrapper(baseDir, stdOutLineConsumer, envVariables, command);
}

static class ProcessWrapper {

private final Path baseDir;
private final Consumer<String> stdOutLineConsumer;
private final String[] command;
private final Map<String, String> envVariables = new HashMap<>();

ProcessWrapper(@Nullable Path baseDir, Consumer<String> stdOutLineConsumer, Map<String, String> envVariables, String... command) {
this.baseDir = baseDir;
this.stdOutLineConsumer = stdOutLineConsumer;
this.envVariables.putAll(envVariables);
this.command = command;
}

private static void processInputStream(InputStream inputStream, Consumer<String> stringConsumer) {
try (var scanner = new Scanner(new InputStreamReader(inputStream, UTF_8))) {
scanner.useDelimiter("\n");
while (scanner.hasNext()) {
stringConsumer.accept(scanner.next());
}
}
}

private static boolean isNotAGitRepo(LinkedList<String> output) {
return output.stream().anyMatch(line -> line.contains("not a git repository"));
}

public void execute() throws IOException {
var output = new LinkedList<String>();
var processBuilder = new ProcessBuilder()
.command(command)
.directory(baseDir != null ? baseDir.toFile() : null);
envVariables.forEach(processBuilder.environment()::put);
processBuilder.redirectInput(ProcessBuilder.Redirect.PIPE);
processBuilder.redirectOutput(ProcessBuilder.Redirect.PIPE);
processBuilder.redirectError(ProcessBuilder.Redirect.PIPE);

var p = processBuilder.start();
try {
processInputStream(p.getInputStream(), line -> {
output.add(line);
stdOutLineConsumer.accept(line);
});

processInputStream(p.getErrorStream(), line -> {
if (!line.isBlank()) {
output.add(line);
LOG.debug(line);
}
});

int exit = p.waitFor();
if (exit != 0) {
if (isNotAGitRepo(output)) {
var dirStr = baseDir != null ? baseDir.toString() : "null";
throw new GitRepoNotFoundException(dirStr);
}
throw new IllegalStateException(format("Command execution exited with code: %d", exit));
}
} catch (InterruptedException e) {
LOG.warn(format("Command [%s] interrupted", join(" ", command)), e);
Thread.currentThread().interrupt();
} finally {
p.destroy();
}
}
}

}
Loading

0 comments on commit d366685

Please sign in to comment.