jk1 · tonylee19544 · Oct 16, 2018
diff --git a/README.md b/README.md
@@ -165,6 +165,49 @@ licenseReport {
 }
 ```
 
+### DownloadLicensesRenderer
+
+The DownloadLicensesRenderer will download all the license files which include url. You can specified licenseUrl to 
+certain license file. When no value pass to DownloadLicenseRenderer the default value is empty Map.
+
+```groovy
+import com.github.jk1.license.render.*
+Map<String, File> customizeLicenseFile = [
+   "http://www.opensource.org/licenses/mit-license.php" : new File("./gradle/default_license/MIT.txt")
+]
+
+licenseReport {
+    renderers = [new DownloadLicensesRenderer(customizeLicenseFile)]
+}
+```
+
+After download all the license files, it will generate a report for all license files report as below. 
+And also a html file containing all html file content inside, which could specified customizeLicenseFile much easier.
+
+```json
+{
+  "noLicenseFileDependencies": [
+    "org.jetbrains.kotlinx:kotlinx-serialization-runtime:0.6.2"
+  ],
+  "noLicenseFileImportedModules": [
+
+  ],
+  "downloadedHtmlLicenseFileDirectories": [
+    "org.projectlombok_lombok_1.16.20/DOWNLOADED-POM-LICENSES/MIT_License.html"
+  ],
+  "downloadedTextLicenseFileDirectories": [
+    "com.couchbase.client_core-io_1.6.1/DOWNLOADED-POM-LICENSES/Apache_License_Version_2_0.txt"
+  ],
+  "embeddedLicenseFileDirectories": [
+    "com.couchbase.client_core-io_1.5.7/META-INF/LICENSE"
+  ]
+}
+```
+```html
+<html><body><div><a href = "http://www.apache.org/licenses/">http://www.apache.org/licenses/</a></div></body></html>
+<html><body><div><a href = "http://www.gnu.org/software/classpath/license.html">http://www.gnu.org/software/classpath/license.html</a></div></body></html>
+<html><body><div><a href = "https://github.com/javaee/javax.annotation/blob/master/LICENSE">https://github.com/javaee/javax.annotation/blob/master/LICENSE</a></div></body></html>
+```
 
 ## Importers
 Importer adds license information from an external source to your report. Importer may come in handy if

diff --git a/src/main/groovy/com/github/jk1/license/reader/LicenseFilesReader.groovy b/src/main/groovy/com/github/jk1/license/reader/LicenseFilesReader.groovy
@@ -20,6 +20,7 @@ import com.github.jk1.license.LicenseFileDetails
 import com.github.jk1.license.LicenseReportExtension
 import com.github.jk1.license.ReportTask
 import com.github.jk1.license.util.Files
+import com.github.jk1.license.util.Paths
 import org.gradle.api.Project
 import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.logging.Logger
@@ -79,7 +80,8 @@ class LicenseFilesReader {
         return entryNames.collect { ZipEntry entry ->
             String entryName = entry.name
             if (!entryName.startsWith("/")) entryName = "/$entryName"
-            String path = "${artifact.file.name}${entryName}"
+            String fileName = Paths.createPathNameFrom(artifact)
+            String path = "${fileName}${entryName}"
             File file = new File(config.outputDir, path)
             file.parentFile.mkdirs()
             file.text = zipFile.getInputStream(entry).text

diff --git a/src/main/groovy/com/github/jk1/license/reader/ManifestReader.groovy b/src/main/groovy/com/github/jk1/license/reader/ManifestReader.groovy
@@ -19,6 +19,7 @@ import com.github.jk1.license.LicenseReportExtension
 import com.github.jk1.license.ManifestData
 import com.github.jk1.license.ReportTask
 import com.github.jk1.license.util.Files
+import com.github.jk1.license.util.Paths
 import org.gradle.api.Project
 import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.logging.Logger
@@ -56,11 +57,13 @@ class ManifestReader {
                 if (mf) {
                     ManifestData data = manifestToData(mf)
                     def path = findLicenseFile(artifact.file, data.license)
-                    if (path != null){
+                    if (path != null) {
                         data.hasPackagedLicense = true
-                        File dest = new File(config.outputDir, "${artifact.file.name}/${data.license}.html")
-                        data.url="${artifact.file.name}/${data.license}.html"
+                        String fileName = Paths.createPathNameFrom(artifact)
+                        String destPath = "${config.outputDir}/${fileName}/EMBEDDED-MANIFEST-LICENSES"
+                        File dest = new File(destPath, data.license)
                         writeLicenseFile(artifact.file, path, dest)
+                        data.url = "${fileName}/EMBEDDED-MANIFEST-LICENSES/${data.license}"
                     }
                     return data
                 }
@@ -113,12 +116,12 @@ class ManifestReader {
         }
     }
 
-    private void writeLicenseFile(File artifactFile, String licenseFileName, File destinationFile) {
+    private void writeLicenseFile(File artifactFile, String licenseFileName, File dest) {
         try {
             ZipFile file = new ZipFile(artifactFile, ZipFile.OPEN_READ)
             ZipEntry entry = file.getEntry(licenseFileName)
-            destinationFile.parentFile.mkdirs()
-            destinationFile.text = file.getInputStream(entry).text
+            dest.parentFile.mkdirs()
+            dest.text = file.getInputStream(entry).text
         } catch (Exception e) {
             LOGGER.warn("Failed to write license file $licenseFileName from $artifactFile", e)
         }

diff --git a/src/main/groovy/com/github/jk1/license/render/DownloadLicensesRenderer.groovy b/src/main/groovy/com/github/jk1/license/render/DownloadLicensesRenderer.groovy
@@ -0,0 +1,222 @@
+/*
+ * Copyright 2018 Evgeny Naumenko <jk.vc@mail.ru>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.github.jk1.license.render
+
+import com.github.jk1.license.ImportedModuleBundle
+import com.github.jk1.license.ImportedModuleData
+import com.github.jk1.license.LicenseFileDetails
+import com.github.jk1.license.LicenseReportExtension
+import com.github.jk1.license.ManifestData
+import com.github.jk1.license.ModuleData
+import com.github.jk1.license.PomData
+import com.github.jk1.license.ProjectData
+import com.github.jk1.license.util.Paths
+import groovy.json.JsonOutput
+
+class DownloadLicensesRenderer implements ReportRenderer {
+    private Map<String, LicenseFileData> licenseUrlsToFileTextCache = [:]
+    private List<File> allLicenseFiles = []
+    private List<File> downloadedHtmlLicenseFiles = []
+    private List<File> downLoadedTextLicenseFiles = []
+    private List<File> embeddedLicenseFiles = []
+    private String htmlFileType = ".html"
+    private String textFileType = ".txt"
+
+    DownloadLicensesRenderer(Map<String, File> customizeLicenseUrlToLicenseFile = [:]) {
+        customizeLicenseUrlToLicenseFile.each {
+            licenseUrlsToFileTextCache.put(it.key, new LicenseFileData(textFileType, it.value.text))
+        }
+    }
+
+    void render(ProjectData projectData) {
+        LicenseReportExtension config= projectData.project.licenseReport
+        String outputDir = config.outputDir
+        downloadStoreAndReportDependenciesLicenses(projectData, outputDir)
+        downloadStoreAndReportImportedModulesLicenses(projectData, outputDir)
+        generateLicenseFileReport(projectData, outputDir)
+        generateAllHtmlUrlFile(outputDir)
+    }
+
+    private void downloadStoreAndReportDependenciesLicenses(ProjectData projectData, String outputDir) {
+        projectData.allDependencies.each { downloadStoreAndReportPomsLicenses(it, outputDir) }
+        projectData.allDependencies.each { downloadStoreAndReportManifestsLicenses(it, outputDir) }
+        projectData.allDependencies.each { reportEmbeddedManifestsLicenses(it, outputDir) }
+        projectData.allDependencies.each { reportEmbeddedLicenseFiles(it, outputDir) }
+    }
+
+    private void downloadStoreAndReportImportedModulesLicenses(ProjectData projectData, String outputDir) {
+        projectData.importedModules.each { downloadStoreAndReportImportedModuleLicenses(it, outputDir) }
+    }
+
+    private void generateLicenseFileReport(ProjectData projectData, String outputDir) {
+        File file = new File("${outputDir}/automatic-included-license-files-report.json")
+        file.createNewFile()
+        finalizeReportData()
+        file.text = JsonOutput.prettyPrint(JsonOutput.toJson([
+            "noLicenseFileDependencies": findAllNoLicenseFilesDependencies(projectData),
+            "noLicenseFileImportedModules": findAllNoLicenseFilesImportedModules(projectData),
+            "downloadedHtmlLicenseFileDirectories": Paths.allRelativePathBetween(downloadedHtmlLicenseFiles.path, outputDir),
+            "downloadedTextLicenseFileDirectories": Paths.allRelativePathBetween(downLoadedTextLicenseFiles.path, outputDir),
+            "embeddedLicenseFileDirectories": Paths.allRelativePathBetween(embeddedLicenseFiles.path, outputDir)
+        ]))
+    }
+
+    private void generateAllHtmlUrlFile(String outputDir) {
+        File file = new File("${outputDir}/automatic-included-license-html-files-contents.html")
+        file.createNewFile()
+        file.text = downloadedHtmlLicenseFiles.text.sort().join()
+    }
+
+    private void downloadStoreAndReportPomsLicenses(ModuleData dependency, String outputDir) {
+        String filePath = Paths.createPathNameFrom(dependency, "DOWNLOADED-POM-LICENSES")
+        dependency.poms.each { downloadStoreAndReportPomLicenses(it, outputDir, filePath) }
+    }
+
+    private void downloadStoreAndReportManifestsLicenses(ModuleData dependency, String outputDir) {
+        String filePath = Paths.createPathNameFrom(dependency, "DOWNLOADED-MANIFEST-LICENSES")
+        dependency.manifests.findAll { it.license }.each { downloadStoreAndReportManifestLicenses(it, outputDir, filePath) }
+    }
+
+    private void reportEmbeddedManifestsLicenses(ModuleData dependency, String outputDir) {
+        dependency.manifests.findAll { it.hasPackagedLicense }.each { reportEmbeddedLicenses(it.url, outputDir) }
+    }
+
+    private void reportEmbeddedLicenseFiles(ModuleData dependency, String outputDir) {
+        dependency.licenseFiles.each { reportEmbeddedFileDetailsLicenses(it.fileDetails, outputDir) }
+    }
+
+    private void downloadStoreAndReportImportedModuleLicenses(ImportedModuleBundle importedModule, String outputDir) {
+        importedModule.modules.findAll {it.licenseUrl}.each {
+            String filePath = Paths.createPathNameFrom(it, "DOWNLOADED-MODULE_LICENSES")
+            downloadStoreAndReportLicenses(it.licenseUrl, it.license, outputDir, filePath)
+        }
+    }
+
+    private void finalizeReportData() {
+        downloadedHtmlLicenseFiles = allLicenseFiles.findAll { it.name.contains(htmlFileType) }
+        downLoadedTextLicenseFiles = allLicenseFiles.findAll { it.path.contains("DOWNLOADED") && it.name.contains(textFileType)}
+        embeddedLicenseFiles = allLicenseFiles.findAll { !it.path.contains("DOWNLOADED") }
+    }
+
+    private List<String> findAllNoLicenseFilesDependencies(ProjectData projectData) {
+        projectData.allDependencies.findAll { isDependencyHasNoLicenseFile(it) }
+            .collect { "${it.group}:${it.name}:${it.version}".toString() }.sort()
+    }
+
+    private List<String> findAllNoLicenseFilesImportedModules(ProjectData projectData) {
+        projectData.importedModules.modules.flatten().findAll { isImportedModuleHasNoLicenseFile(it) }
+            .collect { "${it.name}:${it.version}".toString() }.sort()
+    }
+
+    private void downloadStoreAndReportPomLicenses(PomData pom, String outputDir, String filePath) {
+        pom.licenses.findAll { it.url }.each { downloadStoreAndReportLicenses(it.url, it.name, outputDir, filePath) }
+    }
+
+    private void downloadStoreAndReportManifestLicenses(ManifestData manifest, String outputDir, String filePath) {
+        List<String> licenseUrls = manifest.license.split(/([,])/)*.trim().findAll { !it.contains(" ") }
+        licenseUrls.each { downloadStoreAndReportLicenses(it, it, outputDir, filePath) }
+    }
+
+    private void reportEmbeddedFileDetailsLicenses(Collection<LicenseFileDetails> fileDetails, String outputDir) {
+        fileDetails.collect { it.file }.findAll { it }.each { reportEmbeddedLicenses(it, outputDir) }
+    }
+
+    private boolean isDependencyHasNoLicenseFile(ModuleData dependency) {
+        allLicenseFiles.path.findAll { it.contains(Paths.createPathNameFrom(dependency)) }.empty
+    }
+
+    private boolean isImportedModuleHasNoLicenseFile(ImportedModuleData importedModuleData) {
+        allLicenseFiles.path.findAll { it.contains(Paths.createPathNameFrom(importedModuleData)) }.empty
+    }
+
+    private void downloadStoreAndReportLicenses(String licenseUrl, String licenseName, String outputDir, String filePath) {
+        LicenseFileData fileData = downloadFileTextIfNotInCache(licenseUrl)
+        File newFile = createLicenseFile("${outputDir}/${filePath}", licenseName, fileData.fileType, fileData.fileText)
+        allLicenseFiles.push(newFile)
+    }
+
+    private void reportEmbeddedLicenses(String url, String outputDir) {
+        File localFile = new File("${outputDir}/${url}")
+        allLicenseFiles.push(localFile)
+    }
+
+    private LicenseFileData downloadFileTextIfNotInCache(String url) {
+        return licenseUrlsToFileTextCache.computeIfAbsent(url) { downloadLicenseFileData(url) }
+    }
+
+    private File createLicenseFile(String filePath, String licenseName, String fileType, String fileText) {
+        String fileName = Paths.createFileNameFromLicenseName(licenseName, fileType)
+        File file = new File(filePath, fileName)
+        if (file.exists() && file.text != fileText) file = new File(filePath, fileName.replace(fileType, "_1${fileType}"))
+        file.parentFile.mkdirs()
+        file.createNewFile()
+        file.text = fileText
+        return file
+    }
+
+    private LicenseFileData downloadLicenseFileData(String url) {
+        try {
+            URLConnection urlConnection = new URL(url).openConnection()
+            urlConnection.addRequestProperty("Accept", "text/plain")
+            if (urlConnection.contentType) {
+                if (urlConnection.contentType.contains("text/plain")) {
+                    return new LicenseFileData(textFileType, urlConnection.inputStream.text)
+                }
+            }
+            return new LicenseFileData(htmlFileType, urlToHtmlText(url))
+        } catch (IOException e) {
+            return new LicenseFileData(htmlFileType, urlToHtmlText(url))
+        }
+    }
+
+    private String urlToHtmlText(String url) {
+        return "<html><body><div><a href = \"${url}\">${url}</a></div></body></html>\n"
+    }
+
+    /**this could replace the github url to raw files*/
+    /*private String replaceGitHubLicenseUrl(String licenseUrl) {
+        if (licenseUrl.contains("https://github.com/") && licenseUrl.contains("blob/")) {
+            licenseUrl = licenseUrl.replace("https://github.com", "https://raw.githubusercontent.com")
+            .replace("blob/", "")
+        }
+        return licenseUrl
+    }*/
+
+    /**this could replace the gnu url to txt files*/
+    /*private String replaceGnuLicenseUrl(String licenseUrl) {
+        if (licenseUrl.contains("www.gnu.org") && !licenseUrl.contains("classpath")) {
+            licenseUrl = licenseUrl.replace(".html", ".txt")
+        }
+        return licenseUrl
+    }*/
+
+
+    /**this could add default License Text to certain url*/
+    /*private void addDefaultLicenseText(Map<String, String> customizeLicenseText) {
+        customizeLicenseText.each {
+            licenseUrlsToFileTextCache.put(it.key, it.value)
+        }
+    }*/
+
+    private static class LicenseFileData {
+        String fileType, fileText
+
+        LicenseFileData(String fileType, String fileText) {
+            this.fileType = fileType
+            this.fileText = fileText
+        }
+    }
+}
diff --git a/src/main/groovy/com/github/jk1/license/render/LicenseDataCollector.groovy b/src/main/groovy/com/github/jk1/license/render/LicenseDataCollector.groovy
@@ -18,6 +18,7 @@ package com.github.jk1.license.render
 import com.github.jk1.license.License
 import com.github.jk1.license.LicenseFileDetails
 import com.github.jk1.license.ModuleData
+import com.github.jk1.license.util.Urls
 
 class LicenseDataCollector {
 
@@ -48,7 +49,7 @@ class LicenseDataCollector {
                 info.moduleUrls << manifest.url
             }
             if (manifest.license) {
-                if (isValidUrl(manifest.license)) {
+                if (Urls.isValidUrl(manifest.license)) {
                     if (!info.licenses.find { it.url == manifest.license })
                         info.licenses << new License(url: manifest.license)
                 } else {
@@ -68,15 +69,6 @@ class LicenseDataCollector {
         info
     }
 
-    private static boolean isValidUrl(String url) {
-        try {
-            new URI(url)
-            return true
-        } catch (URISyntaxException e) {
-            return false
-        }
-    }
-
     private static def lastOrNull(Collection l) {
         if (l) l.last() else null
     }

diff --git a/src/main/groovy/com/github/jk1/license/render/ReportRenderer.groovy b/src/main/groovy/com/github/jk1/license/render/ReportRenderer.groovy
@@ -17,7 +17,6 @@ package com.github.jk1.license.render
 
 import com.github.jk1.license.ProjectData
 
-
 interface ReportRenderer {
     void render(ProjectData data)
-}
+}