SLCORE-1086 Investigate git blame slowness

backend/commons/src/main/java/org/sonarsource/sonarlint/core/commons/util/git/BlameParser.java

@@ -0,0 +1,84 @@
+/*
+ * SonarLint Core - Commons
+ * Copyright (C) 2016-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonarsource.sonarlint.core.commons.util.git;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.file.Path;
+import java.text.ParseException;
+import java.util.Date;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.sonar.scm.git.blame.BlameResult;
+import org.sonarsource.sonarlint.core.commons.SonarLintBlameResult;
+
+public class BlameParser {
+
+  public static SonarLintBlameResult parseBlameOutput(String blameOutput, String currentFilePath, Path projectBaseDir) throws IOException, ParseException {
+    BlameResult blameResult = new BlameResult();
+    BlameResult.FileBlame currentFileBlame = new BlameResult.FileBlame(currentFilePath, countCommitterTimeOccurrences(blameOutput));
+    blameResult.getFileBlameByPath().put(currentFilePath, currentFileBlame);
+    String[] fileSections = blameOutput.split("filename ");
+    int currentLineNumber = 0;
+
+    for (String fileSection : fileSections) {
+      if (fileSection.isBlank()) {
+        continue;
+      }
+
+      try (BufferedReader reader = new BufferedReader(new StringReader(fileSection))) {
+        String line;
+
+        while ((line = reader.readLine()) != null) {
+          if (line.startsWith("author ")) {
+            String author = line.substring(7);
+            currentFileBlame.getAuthorEmails()[currentLineNumber] = author;
+          } else if (line.startsWith("author-time ")) {
+            long epochSeconds = Long.parseLong(line.substring(12));
+            Date commitDate = new Date(epochSeconds * 1000);
+            currentFileBlame.getCommitDates()[currentLineNumber] = commitDate;
+          } else if (line.startsWith("commit ")) {
+            String commitHash = line.substring(7);
+            currentFileBlame.getCommitHashes()[currentLineNumber] = commitHash;
+          }
+        }
+      }
+      currentLineNumber++;
+    }
+
+    return new SonarLintBlameResult(blameResult, projectBaseDir);
+  }
+
+  public static int countCommitterTimeOccurrences(String blameOutput) {
+    Pattern pattern = Pattern.compile("^committer-time ", Pattern.MULTILINE);
+    Matcher matcher = pattern.matcher(blameOutput);
+    int count = 0;
+    while (matcher.find()) {
+      count++;
+    }
+    return count;
+  }
+
+  private static int countLines(String blameOutput) {
+    return blameOutput.split("\n").length;
+  }
+
+}

backend/commons/src/main/java/org/sonarsource/sonarlint/core/commons/util/git/GitUtils.java

@@ -19,13 +19,17 @@
  */
 package org.sonarsource.sonarlint.core.commons.util.git;
 
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.net.URI;
 import java.nio.file.Path;
+import java.text.ParseException;
+import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -34,15 +38,18 @@
 import java.util.stream.Stream;
 import javax.annotation.Nullable;
 import org.apache.commons.io.FilenameUtils;
+import org.eclipse.jgit.api.BlameCommand;
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.blame.BlameResult;
 import org.eclipse.jgit.diff.RawTextComparator;
 import org.eclipse.jgit.ignore.IgnoreNode;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.ObjectLoader;
 import org.eclipse.jgit.lib.ObjectReader;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.lib.RepositoryBuilder;
+import org.eclipse.jgit.revwalk.RevCommit;
 import org.eclipse.jgit.revwalk.RevWalk;
 import org.eclipse.jgit.treewalk.TreeWalk;
 import org.sonar.scm.git.blame.RepositoryBlameCommand;
@@ -52,6 +59,7 @@
 
 import static java.util.Optional.ofNullable;
 import static org.eclipse.jgit.lib.Constants.GITIGNORE_FILENAME;
+import static org.sonarsource.sonarlint.core.commons.util.git.BlameParser.parseBlameOutput;
 
 public class GitUtils {
 
@@ -85,6 +93,7 @@ public static List<URI> getVSCChangedFiles(@Nullable Path baseDir) {
   }
 
   public static SonarLintBlameResult blameWithFilesGitCommand(Path projectBaseDir, Set<Path> projectBaseRelativeFilePaths, @Nullable UnaryOperator<String> fileContentProvider) {
+    long startTime = System.currentTimeMillis();
     var gitRepo = buildGitRepository(projectBaseDir);
 
     var gitRepoRelativeProjectBaseDir = getRelativePath(gitRepo, projectBaseDir);
@@ -104,12 +113,85 @@ public static SonarLintBlameResult blameWithFilesGitCommand(Path projectBaseDir,
 
     try {
       var blameResult = blameCommand.call();
+      long endTime = System.currentTimeMillis();
+      LOG.info("blameWithFilesGitCommand took " + (endTime - startTime) + " ms");
       return new SonarLintBlameResult(blameResult, gitRepoRelativeProjectBaseDir);
     } catch (GitAPIException e) {
       throw new IllegalStateException("Failed to blame repository files", e);
     }
   }
 
+  private static boolean isGitAvailable() {
+    try {
+      Process process = new ProcessBuilder("git", "--version").start();
+      int exitCode = process.waitFor();
+      return exitCode == 0;
+    } catch (IOException | InterruptedException e) {
+      return false;
+    }
+  }
+
+  private static String executeGitCommand(Path workingDir, String... command) throws IOException, InterruptedException {
+    var processBuilder = new ProcessBuilder(command);
+    processBuilder.directory(workingDir.toFile());
+    processBuilder.redirectErrorStream(true);
+    var process = processBuilder.start();
+    StringBuilder output = new StringBuilder();
+    try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
+      String line;
+      while ((line = reader.readLine()) != null) {
+        output.append(line).append(System.lineSeparator());
+      }
+    }
+    int exitCode = process.waitFor();
+    if (exitCode != 0) {
+      throw new IOException("Git command failed with exit code " + exitCode);
+    }
+    return output.toString();
+  }
+
+  public static SonarLintBlameResult blameFromNativeCommand(Path projectBaseDir, Set<Path> projectBaseRelativeFilePaths) {
+    if (isGitAvailable()) {
+      long startTime = System.currentTimeMillis();
+      for (var relativeFilePath : projectBaseRelativeFilePaths) {
+        try {
+          var result = parseBlameOutput(executeGitCommand(projectBaseDir, "git", "blame", projectBaseDir.resolve(relativeFilePath).toString(), "--line-porcelain", "--encoding=UTF-8"), projectBaseDir.resolve(relativeFilePath).toString(), projectBaseDir);
+          long endTime = System.currentTimeMillis();
+          LOG.info("blameFromNativeCommand took " + (endTime - startTime) + " ms");
+          return result;
+        } catch (IOException | InterruptedException e) {
+          LOG.debug("Native git command error: ", e);
+        } catch (ParseException e) {
+          throw new IllegalStateException("Failed to blame repository files", e);
+        }
+      }
+    }
+    throw new IllegalStateException("Failed to blame repository files");
+  }
+
+  public static void blameFile(Path projectBaseDir, Set<Path> projectBaseRelativeFilePaths) {
+    long startTime = System.currentTimeMillis();
+    try {
+      var repository = buildGitRepository(projectBaseDir);
+      for (var relativeFilePath : projectBaseRelativeFilePaths) {
+        var blameCommand = new BlameCommand(repository);
+        blameCommand.setFilePath(relativeFilePath.toString());
+        var blameResult = blameCommand.call();
+
+        if (blameResult != null) {
+          for (int i = 0; i < blameResult.getResultContents().size(); i++) {
+            var commit = blameResult.getSourceCommit(i);
+            LOG.info("Line " + (i + 1) + " of file " + relativeFilePath + " was last modified by " + commit.getAuthorIdent().getName() + " on " + commit.getAuthorIdent().getWhen());
+          }
+        }
+      }
+      long endTime = System.currentTimeMillis();
+      LOG.info("blameFile took " + (endTime - startTime) + " ms");
+    } catch (GitAPIException e) {
+      e.printStackTrace();
+    }
+  }
+
   private static Path getRelativePath(Repository gitRepo, Path projectBaseDir) {
     var repoDir = gitRepo.isBare() ? gitRepo.getDirectory() : gitRepo.getWorkTree();
     return repoDir.toPath().relativize(projectBaseDir);

backend/core/src/main/java/org/sonarsource/sonarlint/core/tracking/TrackingService.java

@@ -64,6 +64,8 @@
 import static java.util.Objects.requireNonNull;
 import static java.util.stream.Collectors.toList;
 import static java.util.stream.Collectors.toMap;
+import static org.sonarsource.sonarlint.core.commons.util.git.GitUtils.blameFile;
+import static org.sonarsource.sonarlint.core.commons.util.git.GitUtils.blameFromNativeCommand;
 import static org.sonarsource.sonarlint.core.commons.util.git.GitUtils.blameWithFilesGitCommand;
 
 public class TrackingService {
@@ -268,7 +270,9 @@ private IntroductionDateProvider getIntroductionDateProvider(String configuratio
     var baseDir = getBaseDir(configurationScopeId);
     if (baseDir != null) {
       try {
-        var sonarLintBlameResult = blameWithFilesGitCommand(baseDir, fileRelativePaths, fileContentProvider);
+        var sonarLintBlameResult2 = blameWithFilesGitCommand(baseDir, fileRelativePaths, fileContentProvider);
+        var sonarLintBlameResult = blameFromNativeCommand(baseDir, fileRelativePaths);
+        //blameFile(baseDir, fileRelativePaths);
         return (filePath, lineNumbers) -> determineIntroductionDate(filePath, lineNumbers, sonarLintBlameResult);
       } catch (GitRepoNotFoundException e) {
         LOG.info("Git Repository not found for {}. The path {} is not in a Git repository", configurationScopeId, e.getPath());