Merge pull request #42018 from ShammiL/new-tool-build-integration

Improve tool integration with build implementation

cli/ballerina-cli/src/main/java/io/ballerina/cli/task/CompileTask.java

@@ -29,6 +29,7 @@
 import io.ballerina.projects.ProjectException;
 import io.ballerina.projects.ProjectKind;
 import io.ballerina.projects.SemanticVersion;
+import io.ballerina.projects.buildtools.ToolContext;
 import io.ballerina.projects.directory.SingleFileProject;
 import io.ballerina.projects.environment.ResolutionOptions;
 import io.ballerina.projects.internal.PackageDiagnostic;
@@ -47,6 +48,7 @@
 
 import static io.ballerina.cli.launcher.LauncherUtils.createLauncherException;
 import static io.ballerina.projects.util.ProjectConstants.DOT;
+import static io.ballerina.projects.util.ProjectConstants.TOOL_DIAGNOSTIC_CODE_PREFIX;
 
 /**
  * Task for compiling a package.
@@ -185,7 +187,11 @@ public void execute(Project project) {
                 diagnostics.addAll(project.currentPackage().manifest().diagnostics().diagnostics());
                 // add dependency manifest diagnostics
                 diagnostics.addAll(project.currentPackage().dependencyManifest().diagnostics().diagnostics());
-                diagnostics.forEach(d -> err.println(d.toString()));
+                diagnostics.forEach(d -> {
+                    if (!d.diagnosticInfo().code().startsWith(TOOL_DIAGNOSTIC_CODE_PREFIX)) {
+                        err.println(d);
+                    }
+                });
                 throw createLauncherException("package resolution contains errors");
             }
 
@@ -205,15 +211,24 @@ public void execute(Project project) {
 
             // Report package compilation and backend diagnostics
             diagnostics.addAll(jBallerinaBackend.diagnosticResult().diagnostics(false));
+            diagnostics.forEach(d -> {
+                if (d.diagnosticInfo().code() == null || (!d.diagnosticInfo().code().equals(
+                        ProjectDiagnosticErrorCode.BUILT_WITH_OLDER_SL_UPDATE_DISTRIBUTION.diagnosticId()) &&
+                        !d.diagnosticInfo().code().startsWith(TOOL_DIAGNOSTIC_CODE_PREFIX))) {
+                    err.println(d);
+                }
+            });
+            // Report build tool execution diagnostics
+            if (project.getToolContextMap() != null) {
+                for (ToolContext tool : project.getToolContextMap().values()) {
+                    diagnostics.addAll(tool.diagnostics());
+                }
+            }
             boolean hasErrors = false;
             for (Diagnostic d : diagnostics) {
                 if (d.diagnosticInfo().severity().equals(DiagnosticSeverity.ERROR)) {
                     hasErrors = true;
                 }
-                if (d.diagnosticInfo().code() == null || !d.diagnosticInfo().code().equals(
-                        ProjectDiagnosticErrorCode.BUILT_WITH_OLDER_SL_UPDATE_DISTRIBUTION.diagnosticId())) {
-                    err.println(d);
-                }
             }
             if (hasErrors) {
                 throw createLauncherException("compilation contains errors");

cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunBallerinaPreBuildToolsTask.java

@@ -18,31 +18,38 @@
 
 package io.ballerina.cli.task;
 
-import io.ballerina.cli.tool.CodeGeneratorTool;
 import io.ballerina.cli.utils.FileUtils;
+import io.ballerina.projects.Diagnostics;
 import io.ballerina.projects.Project;
-import io.ballerina.projects.ProjectException;
-import io.ballerina.projects.ToolContext;
+import io.ballerina.projects.buildtools.CodeGeneratorTool;
+import io.ballerina.projects.buildtools.ToolContext;
+import io.ballerina.projects.internal.PackageDiagnostic;
+import io.ballerina.projects.internal.ProjectDiagnosticErrorCode;
 import io.ballerina.toml.api.Toml;
+import io.ballerina.toml.validator.schema.Schema;
 import io.ballerina.tools.diagnostics.Diagnostic;
+import io.ballerina.tools.diagnostics.DiagnosticInfo;
 import io.ballerina.tools.diagnostics.DiagnosticSeverity;
 
 import java.io.IOException;
 import java.io.PrintStream;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.ServiceLoader;
+import java.util.stream.Collectors;
 
 import static io.ballerina.cli.launcher.LauncherUtils.createLauncherException;
 import static io.ballerina.projects.PackageManifest.Tool;
+import static io.ballerina.projects.util.ProjectConstants.TOOL_DIAGNOSTIC_CODE_PREFIX;
 
 /**
  * Task for running tools integrated with the build.
  *
  * @since 2201.9.0
  */
 public class RunBallerinaPreBuildToolsTask implements Task {
-
     private final PrintStream outStream;
 
     public RunBallerinaPreBuildToolsTask(PrintStream out) {
@@ -51,41 +58,74 @@ public RunBallerinaPreBuildToolsTask(PrintStream out) {
 
     @Override
     public void execute(Project project) {
-        Collection<Diagnostic> toolDiagnostics = project.currentPackage().manifest().diagnostics().diagnostics();
-        boolean hasTomlErrors = project.currentPackage().manifest().diagnostics().hasErrors();
-        if (hasTomlErrors) {
-            toolDiagnostics.forEach(outStream::println);
-            throw createLauncherException("compilation contains errors");
-        }
+        // Print all build tool manifest diagnostics
+        Collection<Diagnostic> toolManifestDiagnostics = project.currentPackage().manifest().diagnostics()
+                .diagnostics().stream().filter(diagnostic -> diagnostic.diagnosticInfo().code()
+                .startsWith(TOOL_DIAGNOSTIC_CODE_PREFIX)).collect(Collectors.toList());
+        toolManifestDiagnostics.forEach(outStream::println);
+
+        // Build tool execution
+        Map<String, ToolContext> toolContextMap = new HashMap<>();
         List<Tool> tools = project.currentPackage().manifest().tools();
+        if (!tools.isEmpty()) {
+            this.outStream.println("\nExecuting Build Tools");
+        }
         ServiceLoader<CodeGeneratorTool> buildRunners = ServiceLoader.load(CodeGeneratorTool.class);
         for (Tool tool : tools) {
+            String commandName = tool.type();
+            ToolContext toolContext = ToolContext.from(tool, project.currentPackage());
+            boolean hasOptionErrors = false;
             try {
-                String commandName = tool.getType();
-                CodeGeneratorTool targetTool = getTargetTool(commandName, buildRunners);
-                if (targetTool == null) {
-                    // TODO: Install tool if not found
-                    outStream.println("Command not found: " + commandName);
-                    return;
+                hasOptionErrors = validateOptionsToml(tool.optionsToml(), commandName);
+                if (hasOptionErrors) {
+                    DiagnosticInfo diagnosticInfo = new DiagnosticInfo(
+                            ProjectDiagnosticErrorCode.TOOL_OPTIONS_VALIDATION_FAILED.diagnosticId(),
+                            ProjectDiagnosticErrorCode.TOOL_OPTIONS_VALIDATION_FAILED.messageKey(),
+                            DiagnosticSeverity.ERROR);
+                    PackageDiagnostic diagnostic = new PackageDiagnostic(diagnosticInfo,
+                            tool.type());
+                    toolContext.reportDiagnostic(diagnostic);
+                    toolContextMap.put(tool.id(), toolContext);
                 }
+            } catch (IOException e) {
+                outStream.println(String.format("WARNING: Skipping validation of tool options for tool %s(%s) " +
+                        "due to: %s", tool.type(), tool.id(), e.getMessage()));
+            }
+            if (!tool.hasErrorDiagnostic() && !hasOptionErrors) {
                 try {
-                    validateOptionsToml(tool.getOptionsToml(), commandName);
-                } catch (IOException e) {
-                    outStream.println("WARNING: Skipping the validation of tool options due to: " +
-                        e.getMessage());
-                }
-                ToolContext toolContext = ToolContext.from(tool, project.currentPackage());
-                targetTool.execute(toolContext);
-                toolContext.diagnostics().forEach(outStream::println);
-                for (Diagnostic d : toolContext.diagnostics()) {
-                    if (d.diagnosticInfo().severity().equals(DiagnosticSeverity.ERROR)) {
-                        throw new ProjectException("compilation contains errors");
+                    CodeGeneratorTool targetTool = getTargetTool(commandName, buildRunners);
+                    if (targetTool == null) {
+                        // TODO: Installing tool if not found to be implemented at a later phase
+                        DiagnosticInfo diagnosticInfo = new DiagnosticInfo(
+                                ProjectDiagnosticErrorCode.BUILD_TOOL_NOT_FOUND.diagnosticId(),
+                                "Build tool '" + tool.type() + "' not found",
+                                DiagnosticSeverity.ERROR);
+                        PackageDiagnostic diagnostic = new PackageDiagnostic(diagnosticInfo,
+                                tool.type());
+                        this.outStream.println(diagnostic);
+                        toolContext.reportDiagnostic(diagnostic);
+                        toolContextMap.put(tool.id(), toolContext);
+                        continue;
+                    }
+                    this.outStream.println(String.format("\t%s(%s)%n", tool.type(), tool.id()));
+                    targetTool.execute(toolContext);
+                    for (Diagnostic d : toolContext.diagnostics()) {
+                        if (d.toString().contains("(1:1,1:1)")) {
+                            outStream.println(new PackageDiagnostic(d.diagnosticInfo(), toolContext.toolId()));
+                        } else {
+                            outStream.println(new PackageDiagnostic(d.diagnosticInfo(), d.location()));
+                        }
                     }
+                    toolContextMap.put(tool.id(), toolContext);
+                } catch (Exception e) {
+                    throw createLauncherException(e.getMessage());
                 }
-            } catch (ProjectException e) {
-                throw createLauncherException(e.getMessage());
+            } else {
+                outStream.println(String.format("WARNING: Skipping execution of build tool %s(%s) as Ballerina.toml " +
+                        "contains errors%n", tool.type(), tool.id() != null ? tool.id() : ""));
             }
         }
+        project.setToolContextMap(toolContextMap);
     }
 
     private CodeGeneratorTool getTargetTool(String commandName, ServiceLoader<CodeGeneratorTool> buildRunners) {
@@ -97,16 +137,33 @@ private CodeGeneratorTool getTargetTool(String commandName, ServiceLoader<CodeGe
         return null;
     }
 
-    private void validateOptionsToml(Toml optionsToml, String toolName) throws IOException {
+    private boolean validateOptionsToml(Toml optionsToml, String toolName) throws IOException {
         if (optionsToml == null) {
-            throw new IOException("No tool options found");
+            return validateEmptyOptionsToml(toolName);
         }
         FileUtils.validateToml(optionsToml, toolName);
         optionsToml.diagnostics().forEach(outStream::println);
-        for (Diagnostic d : optionsToml.diagnostics()) {
-            if (d.diagnosticInfo().severity().equals(DiagnosticSeverity.ERROR)) {
-                throw new ProjectException("compilation contains errors");
+        return !Diagnostics.filterErrors(optionsToml.diagnostics()).isEmpty();
+    }
+
+    private boolean validateEmptyOptionsToml(String toolName) throws IOException {
+        Schema schema = Schema.from(io.ballerina.projects.util.FileUtils
+                .readFileAsString(toolName + "-options-schema.json"));
+        List<String> requiredFields = schema.required();
+        if (!requiredFields.isEmpty()) {
+            for (String field: requiredFields) {
+                DiagnosticInfo diagnosticInfo = new DiagnosticInfo(
+                        ProjectDiagnosticErrorCode.MISSING_TOOL_PROPERTIES_IN_BALLERINA_TOML.diagnosticId(),
+                        String.format("missing required optional field '%s'", field),
+                        DiagnosticSeverity.ERROR);
+                PackageDiagnostic diagnostic = new PackageDiagnostic(diagnosticInfo,
+                        toolName);
+                this.outStream.println(diagnostic);
             }
+            return true;
         }
+        this.outStream.println(String.format("WARNING: Skipping validation of tool options for tool %s due to: " +
+                "No tool options found for", toolName));
+        return false;
     }
 }

cli/ballerina-cli/src/main/java/module-info.java

@@ -4,7 +4,6 @@
     exports io.ballerina.cli.launcher;
     exports io.ballerina.cli.utils;
     exports io.ballerina.cli.cmd;
-    exports io.ballerina.cli.tool;
 
     requires io.ballerina.runtime;
     requires io.ballerina.lang;

cli/ballerina-cli/src/test/java/io/ballerina/cli/cmd/BuildCommandTest.java

@@ -1203,9 +1203,7 @@ public void testBuildProjectWithBuildToolTomlPropertyDiagnostics(String projectN
             String buildLog = readOutput(true);
             Assert.assertEquals(buildLog.replaceAll("\r", ""),
                 getOutput(outputFile));
-            Assert.assertTrue(e.getDetailedMessages().get(0)
-                .equals(error));
-
+            Assert.assertEquals(error, e.getDetailedMessages().get(0));
         }
     }
 
@@ -1218,7 +1216,23 @@ public void testBuildProjectWithBuildTool() throws IOException {
         buildCommand.execute();
         String buildLog = readOutput(true);
         Assert.assertEquals(buildLog.replaceAll("\r", ""),
-            getOutput("build-bal-project.txt"));
+            getOutput("build-bal-project-with-build-tool.txt"));
+    }
+
+    @Test(description = "Build a project with a build tool not found")
+    public void testBuildProjectWithBuildToolNotFound() throws IOException {
+        Path projectPath = this.testResources.resolve("build-tool-not-found");
+        System.setProperty(USER_DIR_PROPERTY, projectPath.toString());
+        BuildCommand buildCommand = new BuildCommand(projectPath, printStream, printStream, false);
+        new CommandLine(buildCommand).parseArgs();
+        try {
+            buildCommand.execute();
+        } catch (BLauncherException e) {
+            String buildLog = readOutput(true);
+            Assert.assertEquals(buildLog.replaceAll("\r", ""),
+                    getOutput("build-bal-project-with-build-tool-not-found.txt"));
+            Assert.assertEquals("error: compilation contains errors", e.getDetailedMessages().get(0));
+        }
     }
 
     private String getNewVersionForOldDistWarning() {

cli/ballerina-cli/src/test/resources/test-resources/build-tool-not-found/Ballerina.toml

@@ -0,0 +1,9 @@
+[package]
+org = "foo"
+name = "winery"
+version = "0.1.0"
+
+[[tool.grpc]]
+id = "generate-grpc-client"
+filePath = "delivery_grpc.json"
+targetModule = "delivery"

cli/ballerina-cli/src/test/resources/test-resources/build-tool-not-found/main.bal

@@ -0,0 +1,3 @@
+public function main() {
+    return;
+}

cli/ballerina-cli/src/test/resources/test-resources/build-tool-with-diagnostics/main.bal

@@ -1,5 +1,3 @@
-import ballerina/io;
-
-public function main() {
-    io:println("Hello, World!");
+public function hello() returns string {
+    return "Hello, World!";
 }

cli/ballerina-cli/src/test/resources/test-resources/build-tool-with-invalid-missing-optional-toml-properties/Ballerina.toml

@@ -8,3 +8,8 @@ id = "generate-delivery-client"
 filePath = "delivery.json"
 targetModule = "delivery"
 options.limit = "2"
+
+[[tool.openapi]]
+id = "generate-delivery-client-1"
+filePath = "delivery.json"
+targetModule = "client"

cli/ballerina-cli/src/test/resources/test-resources/build-tool-with-invalid-missing-optional-toml-properties/main.bal

@@ -1,5 +1,3 @@
-import ballerina/io;
-
-public function main() {
-    io:println("Hello, World!");
+public function hello() returns string {
+    return "Hello, World!";
 }

cli/ballerina-cli/src/test/resources/test-resources/build-tool-with-invalid-missing-toml-properties/Ballerina.toml

@@ -6,4 +6,9 @@ version = "0.1.0"
 [[tool.openapi]]
 id = "generate-delivery-client"
 filePath = ""
+targetModule = "delivery"
+options.mode = "client"
+
+[[tool.openapi]]
+id = "generate-client"
 options.mode = "client"

cli/ballerina-cli/src/test/resources/test-resources/build-tool-with-invalid-missing-toml-properties/main.bal

@@ -1,5 +1,3 @@
-import ballerina/io;
-
-public function main() {
-    io:println("Hello, World!");
+public function hello() returns string {
+    return "Hello, World!";
 }

cli/ballerina-cli/src/test/resources/test-resources/build-tool-with-recurring-tool-properties/Ballerina.toml

@@ -6,15 +6,12 @@ version = "0.1.0"
 [[tool.openapi]]
 id = "generate-delivery-client"
 filePath = "delivery.json"
-targetModule = "delivery"
+targetModule = ""
 options.mode = "client"
 
 [[tool.openapi]]
 id = "generate-delivery-client"
-filePath = "delivery1.json"
+filePath = "delivery.yml"
 options.mode = "client"
 
-[[tool.grpc]]
-id = "generate-grpc-client"
-filePath = "delivery2.json"
-targetModule = "delivery"
+

cli/ballerina-cli/src/test/resources/test-resources/build-tool-with-recurring-tool-properties/main.bal

@@ -1,3 +1,3 @@
-public function main() {
+public function hello() returns string {
     return "Hello, World!";
 }

cli/ballerina-cli/src/test/resources/test-resources/command-outputs/unix/build-bal-project-with-build-tool-not-found.txt

@@ -0,0 +1,6 @@
+
+Executing Build Tools
+WARNING: Skipping validation of tool options for tool grpc(generate-grpc-client) due to: Schema file not found: grpc-options-schema.json
+ERROR [grpc] Build tool 'grpc' not found
+Compiling source
+	foo/winery:0.1.0

cli/ballerina-cli/src/test/resources/test-resources/command-outputs/unix/build-bal-project-with-build-tool.txt

@@ -0,0 +1,9 @@
+
+Executing Build Tools
+	openapi(generate-delivery-client)
+
+Compiling source
+	foo/winery:0.1.0
+
+Generating executable
+	target/bin/winery.jar

cli/ballerina-cli/src/test/resources/test-resources/command-outputs/unix/build-tool-with-diagnostics.txt

@@ -1 +1,7 @@
-ERROR [openAPI sample build tool:(1:1,1:1)] The provided filePath does not exist
+
+Executing Build Tools
+	openapi(generate-delivery-client)
+
+ERROR [generate-delivery-client] The provided filePath does not exist
+Compiling source
+	foo/winery:0.1.0