Merge pull request #845 from Cosmo-Tech/DSE/further_upgrades_PROD-14136

Dse/further upgrades prod 14136

Author: sellisd

README.md

@@ -359,6 +359,26 @@ kube-linter --config api/kubernetes/.kube-linter.yaml lint api/kubernetes/helm-c
 kube-linter --config api/kubernetes/.kube-linter.yaml lint api/kubernetes/csm-argo
 ```
 
+#### Vulnerability report
+
+To generate a report of publicly disclosed vulnerabilities in the dependencies
+add your API key for the National Vulnerability Database (https://nvd.nist.gov/)
+as a property available to gradle. If you don't have a key get one from
+here: https://nvd.nist.gov/developers/request-an-api-key. Add your key in your
+`~/.gradle/gradle.properties` file (create the file if it does not exist)
+
+```properties
+NVD_API_key=[key]
+```
+
+Then run the dependency check task which can take about 10 minutes:
+
+```shell
+./gradlew dependencyCheckAggregate
+```
+
+an html report will be generated under `/build/reports`
+
 ## License
 
     Copyright 2021 Cosmo Tech

build.gradle.kts

@@ -3,12 +3,10 @@
 import com.diffplug.gradle.spotless.SpotlessExtension
 import com.github.jk1.license.filter.LicenseBundleNormalizer
 import com.github.jk1.license.render.*
-import com.github.jk1.license.task.CheckLicenseTask
 import com.github.jk1.license.task.ReportTask
 import com.google.cloud.tools.jib.api.buildplan.ImageFormat.OCI
 import com.google.cloud.tools.jib.gradle.JibExtension
 import io.gitlab.arturbosch.detekt.Detekt
-import java.io.FileOutputStream
 import org.apache.tools.ant.filters.ReplaceTokens
 import org.gradle.api.tasks.testing.logging.TestExceptionFormat
 import org.gradle.api.tasks.testing.logging.TestLogEvent
@@ -29,13 +27,13 @@ plugins {
   kotlin("jvm") version kotlinVersion
   kotlin("plugin.spring") version kotlinVersion apply false
   id("pl.allegro.tech.build.axion-release") version "1.15.5"
-  id("com.diffplug.spotless") version "6.22.0"
-  id("org.springframework.boot") version "3.3.6" apply false
+  id("com.diffplug.spotless") version "6.25.0"
+  id("org.springframework.boot") version "3.4.1" apply false
   id("project-report")
-  id("org.owasp.dependencycheck") version "9.0.2"
-  id("com.github.jk1.dependency-license-report") version "2.5"
+  id("org.owasp.dependencycheck") version "12.0.0"
+  id("com.github.jk1.dependency-license-report") version "2.9"
   id("org.jetbrains.kotlinx.kover") version "0.7.4"
-  id("io.gitlab.arturbosch.detekt") version "1.23.6"
+  id("io.gitlab.arturbosch.detekt") version "1.23.7"
   id("org.openapi.generator") version "7.8.0" apply false
   id("com.google.cloud.tools.jib") version "3.4.4" apply false
 }
@@ -50,28 +48,28 @@ version = scmVersion.version
 
 // Required versions
 val jacksonVersion = "2.15.3"
-val springWebVersion = "6.1.16"
+val springWebVersion = "6.2.1"
 
 // Implementation
 val kotlinJvmTarget = 21
 val cosmotechApiCommonVersion = "2.1.0-SNAPSHOT"
 val jedisVersion = "4.4.6"
-val springOauthVersion = "6.2.2"
-val redisOmSpringVersion = "0.9.1"
+val springOauthVersion = "6.4.2"
+val redisOmSpringVersion = "0.9.7"
 val kotlinCoroutinesCoreVersion = "1.8.1"
-val oktaSpringBootVersion = "3.0.5"
+val oktaSpringBootVersion = "3.0.7"
 val springDocVersion = "2.5.0"
-val swaggerParserVersion = "2.1.22"
+val swaggerParserVersion = "2.1.24"
 val commonsCsvVersion = "1.10.0"
 val apiValidationVersion = "3.0.2"
 val kubernetesClientVersion = "22.0.0"
 
 // Checks
-val detektVersion = "1.23.6"
+val detektVersion = "1.23.7"
 
 // Tests
 val jUnitBomVersion = "5.10.0"
-val mockkVersion = "1.13.8"
+val mockkVersion = "1.13.13"
 val awaitilityKVersion = "4.2.0"
 val testcontainersRedis = "1.6.4"
 val springMockkVersion = "4.0.2"
@@ -82,38 +80,33 @@ val configBuildDir = "${layout.buildDirectory.get()}/config"
 
 mkdir(configBuildDir)
 
-fun downloadLicenseConfigFile(name: String): String {
-  val localPath = "$configBuildDir/$name"
-  val f = file(localPath)
-  f.delete()
-  val url = "https://raw.githubusercontent.com/Cosmo-Tech/cosmotech-license/main/config/$name"
-  logger.info("Downloading license config file from $url to $localPath")
-  uri(url).toURL().openStream().use { it.copyTo(FileOutputStream(f)) }
-  return localPath
-}
-
-val licenseNormalizerPath = downloadLicenseConfigFile("license-normalizer-bundle.json")
-val licenseAllowedPath =
-    if (project.properties["useLocalLicenseAllowedFile"] == "true") {
-      "$projectDir/config/allowed-licenses.json"
-    } else {
-      downloadLicenseConfigFile("allowed-licenses.json")
-    }
-
-logger.info("Using licenses allowed file: $licenseAllowedPath")
-
-val licenseEmptyPath = downloadLicenseConfigFile("empty-dependencies-resume.json")
-// Plugin uses a generated report to check the licenses in a prepation task
 val hardCodedLicensesReportPath = "project-licenses-for-check-license-task.json"
 
+dependencyCheck{
+  // Configure dependency check plugin. It checks for publicly disclosed
+  // vulnerabilities in project dependencies. To use it, you need to have an
+  // API key from the NVD (National Vulnerability Database), pass it by setting
+  // the environment variable NVD_API_key (See project README.md: Static Code
+  // Analysis -> Vulnerability report).
+  nvd{
+    apiKey = System.getenv("NVD_API_key")
+  }
+}
+
 licenseReport {
   outputDir = licenseReportDir
-  allowedLicensesFile = file(licenseAllowedPath)
+  allowedLicensesFile =
+      "https://raw.githubusercontent.com/Cosmo-Tech/cosmotech-license/refs/heads/main/config/allowed-licenses.json"
+  val bundle =
+      "https://raw.githubusercontent.com/Cosmo-Tech/cosmotech-license/refs/heads/main/config/license-normalizer-bundle.json"
+
   renderers =
       arrayOf<ReportRenderer>(
           InventoryHtmlReportRenderer("index.html"),
           JsonReportRenderer("project-licenses-for-check-license-task.json", false))
-  filters = arrayOf<LicenseBundleNormalizer>(LicenseBundleNormalizer(licenseNormalizerPath, true))
+  filters =
+      arrayOf<LicenseBundleNormalizer>(
+          LicenseBundleNormalizer(uri(bundle).toURL().openStream(), true))
 }
 
 allprojects {
@@ -169,12 +162,10 @@ allprojects {
       licenseHeader(licenseHeaderComment)
     }
     kotlin {
-      ktfmt("0.41")
       target("**/*.kt")
       licenseHeader(licenseHeaderComment)
     }
     kotlinGradle {
-      ktfmt("0.41")
       target("**/*.kts")
       //      licenseHeader(licenseHeaderComment, "import")
     }
@@ -288,9 +279,8 @@ subprojects {
     implementation("org.springdoc:springdoc-openapi-starter-webmvc-ui:${springDocVersion}")
     implementation("io.swagger.parser.v3:swagger-parser-v3:${swaggerParserVersion}")
     implementation("org.springframework.boot:spring-boot-starter-security")
-    implementation("org.springframework.security:spring-security-oauth2-jose:${springOauthVersion}")
-    implementation(
-        "org.springframework.security:spring-security-oauth2-resource-server:${springOauthVersion}")
+    implementation("org.springframework.security:spring-security-oauth2-jose")
+    implementation("org.springframework.security:spring-security-oauth2-resource-server")
     implementation("com.okta.spring:okta-spring-boot-starter:${oktaSpringBootVersion}")
 
     implementation("org.apache.commons:commons-csv:$commonsCsvVersion")
@@ -558,41 +548,7 @@ extensions.configure<kotlinx.kover.gradle.plugin.dsl.KoverReportExtension> {
 }
 
 // https://github.com/jk1/Gradle-License-Report/blob/master/README.md
-tasks.register<ReportTask>("generateLicenseDoc") {}
-
-tasks.register<CheckLicenseTask>("validateLicense") {
-  dependsOn("generateLicenseDoc")
-  // Gradle task must be rerun each time to take new allowed-license into account.
-  // Due to an issue in the plugin, we must define each module name for null licenses
-  // to avoid false negatives in the allowed-license file.
-  outputs.upToDateWhen { false }
-}
+tasks.register<ReportTask>(
+    "generateLicenseDoc") {}
 
-tasks.register("displayLicensesNotAllowed") {
-  val notAllowedFile =
-      file(
-          buildString {
-            append(licenseReportDir)
-            append("/dependencies-without-allowed-license.json")
-          })
-  val dependenciesEmptyResumeTemplate = file(licenseEmptyPath)
-  if (notAllowedFile.exists() && dependenciesEmptyResumeTemplate.exists()) {
-    if (notAllowedFile.readText() != dependenciesEmptyResumeTemplate.readText()) {
-      logger.warn("Licenses not allowed:")
-      logger.warn(notAllowedFile.readText())
-      logger.warn(
-          "Please review licenses and add new license check rules in https://github.com/Cosmo-Tech/cosmotech-license")
-    } else {
-      logger.warn("No error in licences detected!")
-    }
-  }
-}
 
-gradle.buildFinished {
-  if (project.properties["skipLicenses"] != "true") {
-    val validateLicenseTask = tasks.getByName("validateLicense")
-    validateLicenseTask.run {}
-    val displayTask = tasks.getByName("displayLicensesNotAllowed")
-    displayTask.run {}
-  }
-}

dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceIntegrationTest.kt

@@ -477,6 +477,7 @@ class DatasetServiceIntegrationTest : CsmRedisTestBase() {
     assertEquals(1, datasetList.size)
     assertEquals(datasetNotReachableByCurrentUserBecausePartOfAnotherOrganization, datasetList[0])
   }
+
   @Test
   fun `test find All Datasets with wrong pagination params`() {
     organizationSaved = organizationApiService.registerOrganization(organization)
@@ -1108,6 +1109,7 @@ class DatasetServiceIntegrationTest : CsmRedisTestBase() {
                         OrganizationAccessControl(id = CONNECTED_ADMIN_USER, role = ROLE_ADMIN),
                         OrganizationAccessControl(id = userName, role = role))))
   }
+
   fun makeDataset(
       organizationId: String = organizationSaved.id!!,
       parentId: String = "",

dataset/src/integrationTest/kotlin/com/cosmotech/dataset/service/DatasetServiceRBACTest.kt

@@ -2312,6 +2312,7 @@ class DatasetServiceRBACTest : CsmRedisTestBase() {
         ioTypes = listOf(),
         id = "c-AbCdEf123")
   }
+
   fun makeDataset(
       id: String,
       name: String,

dataset/src/main/kotlin/com/cosmotech/dataset/DatasetApiServiceInterface.kt

@@ -23,6 +23,7 @@ interface DatasetApiServiceInterface : DatasetApiService {
   /**
    * Find Dataset by Organization Id and Dataset Id (checking ONLY READ_PERMISSION on targeted
    * organization)
+   *
    * @param organizationId an organization Id
    * @param datasetId a dataset Id
    * @return a Dataset or null
@@ -31,6 +32,7 @@ interface DatasetApiServiceInterface : DatasetApiService {
 
   /**
    * Add a new entry (ou update existing one) on dataset passed in parameter
+   *
    * @param organizationId an organization id
    * @param dataset a dataset to update
    * @param identity a user/application identity

dataset/src/main/kotlin/com/cosmotech/dataset/bulk/BulkQuery.kt

@@ -21,21 +21,25 @@ private constructor(
       private var typedBinaryEdges: Map<String, BinaryEntities> = mapOf()
   ) {
     fun graphName(name: String) = apply { this.graphName = name }
+
     fun first() = apply { this.begin = true }
 
     fun addNodeTypeGroup(type: String, binaryEntities: BinaryEntities) = apply {
       if (type !in typedBinaryNodes) this.typedBinaryNodes += type to binaryEntities
     }
+
     fun addNodeToNodeTypeGroup(type: String, node: Node) = apply {
       if (type in this.typedBinaryNodes) this.typedBinaryNodes[type]?.addEntity(node)
     }
 
     fun addEdgeTypeGroup(type: String, typeEdge: BinaryEntities) = apply {
       if (type !in typedBinaryEdges) this.typedBinaryEdges += type to typeEdge
     }
+
     fun addEdgeToEdgeTypeGroup(type: String, edge: Edge) = apply {
       if (type in this.typedBinaryEdges) this.typedBinaryEdges[type]?.addEntity(edge)
     }
+
     // For ease, those sizes are not calculated as there are very small
     //   GRAPH.BULK     string to byteArray size
     //   graphName      string to byteArray size
@@ -73,6 +77,7 @@ private constructor(
 
   private fun ByteArray.toHex(): String =
       joinToString(separator = "") { eachByte -> "%02x".format(eachByte) }
+
   override fun toString(): String {
     return generateQueryArgs().joinToString(separator = " - ") { bytes -> bytes.toHex() }
   }

dataset/src/main/kotlin/com/cosmotech/dataset/bulk/QueryBuffer.kt

@@ -27,6 +27,7 @@ class QueryBuffer(val unifiedJedis: UnifiedJedis, val graphName: String) {
 
   private val nodeSet: HashSet<String> = hashSetOf()
   private val edgeSet: HashSet<String> = hashSetOf()
+
   fun addNode(type: String, id: String, properties: Map<String, Any?>) {
     addEntity(type, properties) {
       if (id in nodeSet) return@addEntity null

dataset/src/main/kotlin/com/cosmotech/dataset/service/DatasetServiceImpl.kt

@@ -194,9 +194,7 @@ class DatasetServiceImpl(
     dataset.takeUnless {
       datasetSourceType in listOf(DatasetSourceType.ADT, DatasetSourceType.AzureStorage) &&
           dataset.source == null
-    }
-        ?: throw IllegalArgumentException(
-            "Source cannot be null for source type 'ADT' or 'Storage'")
+    } ?: throw IllegalArgumentException("Source cannot be null for source type 'ADT' or 'Storage'")
 
     var twingraphId: String? = null
 
@@ -1004,8 +1002,7 @@ class DatasetServiceImpl(
     val datasetCompatibilityMap =
         existingDataset.compatibility
             ?.associateBy { "${it.solutionKey}-${it.minimumVersion}-${it.maximumVersion}" }
-            ?.toMutableMap()
-            ?: mutableMapOf()
+            ?.toMutableMap() ?: mutableMapOf()
     datasetCompatibilityMap.putAll(
         datasetCompatibility
             .filter { it.solutionKey.isNotBlank() }

dataset/src/main/kotlin/com/cosmotech/dataset/utils/Twingraph.ext.kt

@@ -28,6 +28,7 @@ enum class CsmGraphEntityType {
   RELATION,
   NODE
 }
+
 // To support simple quoted jsonstring from ADT
 val jsonObjectMapper: ObjectMapper =
     objectMapper().configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true)
@@ -41,7 +42,8 @@ fun GraphEntity.toCsmGraphEntity(type: CsmGraphEntityType): CsmGraphEntity {
   val label =
       when (type) {
         CsmGraphEntityType.RELATION -> (this as Edge).relationshipType
-        CsmGraphEntityType.NODE -> (this as Node).takeIf { it.numberOfLabels > 0 }?.getLabel(0)
+        CsmGraphEntityType.NODE ->
+            (this as Node).takeIf { it.numberOfLabels > 0 }?.getLabel(0)
                 ?: throw IndexOutOfBoundsException("Node has no label: $entityId")
       }
 

doc/licenses/HdrHistogram-2.2.2.jar/META-INF/LICENSE.txt

@@ -0,0 +1,41 @@
+The code in this repository code was Written by Gil Tene, Michael Barker,
+and Matt Warren, and released to the public domain, as explained at
+http://creativecommons.org/publicdomain/zero/1.0/
+
+For users of this code who wish to consume it under the "BSD" license
+rather than under the public domain or CC0 contribution text mentioned
+above, the code found under this directory is *also* provided under the
+following license (commonly referred to as the BSD 2-Clause License). This
+license does not detract from the above stated release of the code into
+the public domain, and simply represents an additional license granted by
+the Author.
+
+-----------------------------------------------------------------------------
+** Beginning of "BSD 2-Clause License" text. **
+
+ Copyright (c) 2012, 2013, 2014, 2015, 2016 Gil Tene
+ Copyright (c) 2014 Michael Barker
+ Copyright (c) 2014 Matt Warren
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ THE POSSIBILITY OF SUCH DAMAGE.