License report & automatic LICENSE content check (apache#54)

NOTICE

@@ -4,3 +4,11 @@ Copyright 2024 The Apache Software Foundation
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
 
+This product includes software licensed under the terms
+of the following licenses, see LICENSE file:
+* Apache Software License, Version 2.0
+* Creative Commons 1.0 Universal
+* Eclipse Distribution License, Version 1.0
+* Eclipse Public License, Version 1.0
+* Eclipse Public License, Version 2.0
+* GNU General Public License, Version 2 with the GNU Classpath Exception

aggregated-license-report/build.gradle.kts

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.gradle.kotlin.dsl.support.unzipTo
+
+val licenseReports by configurations.creating { description = "Used to generate license reports" }
+
+dependencies {
+  licenseReports(project(":polaris-service", "licenseReports"))
+}
+
+val collectLicenseReportJars by
+  tasks.registering(Sync::class) {
+    destinationDir = project.layout.buildDirectory.dir("tmp/license-report-jars").get().asFile
+    from(licenseReports)
+  }
+
+val aggregateLicenseReports by
+  tasks.registering {
+    group = "Build"
+    description = "Aggregates license reports"
+    val outputDir = project.layout.buildDirectory.dir("licenseReports")
+    outputs.dir(outputDir)
+    dependsOn(collectLicenseReportJars)
+    doLast {
+      delete(outputDir)
+      fileTree(collectLicenseReportJars.get().destinationDir).files.forEach { zip ->
+        val targetDirName = zip.name.replace("-license-report.zip", "")
+        unzipTo(outputDir.get().dir(targetDirName).asFile, zip)
+      }
+    }
+  }
+
+val aggregatedLicenseReportsZip by
+  tasks.registering(Zip::class) {
+    from(aggregateLicenseReports)
+    from(rootProject.layout.projectDirectory) {
+      include("NOTICE", "LICENSE")
+      eachFile {
+        path = file.name + ".txt"
+      }
+    }
+    archiveBaseName.set("polaris-aggregated-license-report-${project.version}")
+    destinationDirectory.set(layout.buildDirectory.dir("distributions"))
+    archiveExtension.set("zip")
+  }

build-logic/build.gradle.kts

@@ -23,5 +23,6 @@ dependencies {
   implementation(gradleKotlinDsl())
   implementation(baselibs.errorprone)
   implementation(baselibs.idea.ext)
+  implementation(baselibs.license.report)
   implementation(baselibs.spotless)
 }

build-logic/src/main/kotlin/LicenseFileValidation.kt

@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import com.github.jk1.license.LicenseReportExtension
+import com.github.jk1.license.ProjectData
+import com.github.jk1.license.filter.DependencyFilter
+import java.io.File
+import org.gradle.api.GradleException
+
+/**
+ * Validates that all dependencies with MIT/BSD/Go/UPL/ISC licenses, and Apache
+ * license, are mentioned in the `LICENSE` file.
+ */
+class LicenseFileValidation : DependencyFilter {
+  val needsApacheLicenseMention = setOf("Apache")
+
+  val needsFullLicenseMention = setOf("MIT", "BSD", "Go", "ISC", "Universal Permissive")
+
+  fun doesNeedApacheMention(licenses: List<String?>): Boolean {
+    for (license in licenses) {
+      if (license != null) {
+        if (needsApacheLicenseMention.any { license.contains(it) }) {
+          return true
+        }
+      }
+    }
+    return false
+  }
+
+  fun doesNeedFullMention(licenses: List<String?>): Boolean {
+    for (license in licenses) {
+      if (license != null) {
+        if (needsFullLicenseMention.any { license.contains(it) }) {
+          return true
+        }
+      }
+    }
+    // no licenses !
+    return true
+  }
+
+  override fun filter(data: ProjectData?): ProjectData {
+    data!!
+
+    val rootLicenseFile = data.project.rootProject.file("LICENSE").readText()
+
+    val licenseReport = data.project.extensions.getByType(LicenseReportExtension::class.java)
+
+    val missingApacheMentions = mutableSetOf<String>()
+    val missingFullMentions = mutableMapOf<String, String>()
+
+    data.allDependencies.forEach { mod ->
+      val licenses =
+        (mod.manifests.map { it.license } +
+            mod.licenseFiles.flatMap { it.fileDetails }.map { it.license } +
+            mod.poms.flatMap { it.licenses }.map { it.name })
+          .distinct()
+
+      val groupModule = "${mod.group}:${mod.name}"
+      val groupModuleRegex = "^$groupModule$".toRegex(RegexOption.MULTILINE)
+      if (!groupModuleRegex.containsMatchIn(rootLicenseFile)) {
+        if (doesNeedApacheMention(licenses)) {
+          missingApacheMentions.add(groupModule)
+        } else if (doesNeedFullMention(licenses)) {
+            missingFullMentions[groupModule] = """
+            ---
+            $groupModule
+
+            ${mod.licenseFiles.flatMap { it.fileDetails }.filter { it.file != null }.map { it.file }
+              .map { File("${licenseReport.absoluteOutputDir}/$it").readText().trim() }
+              .distinct()
+              .map { "\n\n$it\n" }
+              .joinToString("\n")
+            }
+            """.trimIndent()
+        }
+      }
+    }
+
+    val missingError = StringBuilder()
+    if (!missingApacheMentions.isEmpty()) {
+      missingError.append("\n\nMissing Apache License mentions:")
+      missingError.append("\n--------------------------------\n")
+      missingApacheMentions.sorted().forEach {
+        missingError.append("\n$it")
+      }
+    }
+    if (!missingFullMentions.isEmpty()) {
+      missingError.append("\n\nMissing full license mentions:")
+      missingError.append("\n------------------------------\n")
+      missingFullMentions.toSortedMap().values.forEach {
+        missingError.append("\n$it")
+      }
+    }
+    if (!missingApacheMentions.isEmpty() || !missingFullMentions.isEmpty()) {
+
+      throw GradleException(
+        "License information for the following artifacts is missing in the root LICENSE file: $missingError"
+      )
+    }
+
+    return data
+  }
+}

build-logic/src/main/kotlin/polaris-license-report.gradle.kts

@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import com.github.jk1.license.filter.LicenseBundleNormalizer
+import com.github.jk1.license.render.InventoryHtmlReportRenderer
+import com.github.jk1.license.render.JsonReportRenderer
+import com.github.jk1.license.render.XmlReportRenderer
+import java.util.*
+
+plugins { id("com.github.jk1.dependency-license-report") }
+
+afterEvaluate {
+  licenseReport {
+    filters =
+      arrayOf(
+        LicenseBundleNormalizer(
+          "${rootProject.projectDir}/gradle/license/normalizer-bundle.json",
+          false
+        ),
+        LicenseFileValidation()
+      )
+    allowedLicensesFile = rootProject.projectDir.resolve("gradle/license/allowed-licenses.json")
+    renderers =
+      arrayOf(InventoryHtmlReportRenderer("index.html"), JsonReportRenderer(), XmlReportRenderer())
+    excludeBoms = true
+    outputDir = "${project.layout.buildDirectory.get()}/reports/dependency-license"
+  }
+}
+
+val generateLicenseReport =
+  tasks.named("generateLicenseReport") {
+    inputs
+      .files(
+        rootProject.projectDir.resolve("gradle/license/normalizer-bundle.json"),
+        rootProject.projectDir.resolve("gradle/license/allowed-licenses.json")
+      )
+      .withPathSensitivity(PathSensitivity.RELATIVE)
+    inputs.property("renderersHash", Arrays.hashCode(licenseReport.renderers))
+    inputs.property("filtersHash", Arrays.hashCode(licenseReport.filters))
+    inputs.property("excludesHash", Arrays.hashCode(licenseReport.excludes))
+    inputs.property("excludeGroupsHash", Arrays.hashCode(licenseReport.excludeGroups))
+  }
+
+val licenseReportZip =
+  tasks.register<Zip>("licenseReportZip") {
+    group = "documentation"
+    description = "License report as a ZIP"
+    dependsOn("checkLicense")
+    from(generateLicenseReport)
+    archiveClassifier.set("license-report")
+    archiveExtension.set("zip")
+  }
+
+val licenseReports by
+  configurations.creating {
+    isCanBeConsumed = true
+    isCanBeResolved = false
+    description = "License report files"
+    outgoing { artifact(licenseReportZip) }
+  }
+
+plugins.withType<MavenPublishPlugin>().configureEach {
+  configure<PublishingExtension> {
+    publications { named<MavenPublication>("maven") { artifact(licenseReportZip) } }
+  }
+}
+
+tasks.named("check") { dependsOn(generateLicenseReport) }

gradle/baselibs.versions.toml

@@ -20,4 +20,5 @@
 [libraries]
 errorprone = { module = "net.ltgt.gradle:gradle-errorprone-plugin", version = "4.0.1" }
 idea-ext = { module = "gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext", version = "1.1.8" }
+license-report = { module = "com.github.jk1:gradle-license-report", version = "2.8" }
 spotless = { module = "com.diffplug.spotless:spotless-plugin-gradle", version = "6.25.0" }

gradle/libs.versions.toml

@@ -28,6 +28,10 @@ swagger = "1.6.14"
 
 
 [libraries]
+#
+# If a dependency is removed, check whether the LICENSE and/or NOTICE files need to be adopted
+# (aka mention of the dependency removed).
+#
 assertj-core = { module = "org.assertj:assertj-core", version = "3.26.3" }
 auth0-jwt = { module = "com.auth0:java-jwt", version = "4.2.1" }
 awssdk-bom = { module = "software.amazon.awssdk:bom", version = "2.26.25" }